fuzz on power (#1326)

* first try

* use ubuntu 20.04, do the fuzzing

* new try at power fuzz

* hard code clang version

* setting env variables does not seem to work

* use fuzzer-no-link

* switch to Debian Buster for power fuzz

* use non-sanitizer build for power

* me not like yaml

* fix bad syntax

.github/workflows/power-fuzz.yml

@@ -0,0 +1,62 @@
+name: short fuzz on the power arch
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  armv7_job:
+    # The host should always be Linux
+    runs-on: ubuntu-20.04
+    name: Build on ubuntu-20.04 ppc64le
+    steps:
+      - uses: actions/checkout@v2.1.0
+      - uses: uraimo/run-on-arch-action@v2.0.5
+        name: Run commands
+        id: runcmd
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        with:
+          arch: ppc64le
+          distro: buster
+
+          # Not required, but speeds up builds by storing container images in
+          # a GitHub package registry.
+          githubToken: ${{ github.token }}
+
+          run: |
+            export CLANGSUFFIX="-7"
+            apt-get -qq update
+            apt-get install -q -y clang-7 libfuzzer-7-dev cmake git wget zip ninja-build
+            mkdir -p build ; cd build
+            cmake .. -GNinja \
+                  -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX \
+                  -DCMAKE_C_COMPILER=clang$CLANGSUFFIX \
+                  -DSIMDJSON_BUILD_STATIC=Off \
+                  -DENABLE_FUZZING=On \
+                  -DSIMDJSON_COMPETITION=OFF \
+                  -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \
+                  -DSIMDJSON_DISABLE_DEPRECATED_API=On \
+                  -DSIMDJSON_FUZZ_LDFLAGS=-lFuzzer \
+                  -DCMAKE_CXX_FLAGS="-fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=" \
+                  -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link" \
+                  -DCMAKE_BUILD_TYPE=Release \
+                  -DSIMDJSON_FUZZ_LINKMAIN=Off
+            cd ..
+            builddir=build
+            cmake --build $builddir
+            wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar
+            tar xf corpus.tar && rm corpus.tar
+            fuzzernames=$(cmake --build $builddir --target print_all_fuzzernames |tail -n1)
+            for fuzzer in $fuzzernames ; do
+               exe=$builddir/fuzz/$fuzzer
+               shortname=$(echo $fuzzer |cut -f2- -d_)
+               echo found fuzzer $shortname with executable $exe
+               mkdir -p out/$shortname
+               others=$(find out -type d -not -name $shortname -not -name out -not -name cmin)
+               $exe -max_total_time=20  -max_len=4000 out/$shortname $others
+               echo "*************************************************************************"
+            done
+            echo "all is good, no errors found in any of these fuzzers: $fuzzernames"

fuzz/CMakeLists.txt

@@ -35,9 +35,9 @@ if(ENABLE_FUZZING)
     target_sources(simdjson-fuzzer INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/main.cpp)
   else ()
     target_link_libraries(simdjson-fuzzer INTERFACE simdjson)
+    target_link_libraries(simdjson-fuzzer INTERFACE ${SIMDJSON_FUZZ_LDFLAGS})
   endif ()
   target_link_libraries(simdjson-fuzzer INTERFACE simdjson-internal-flags)
-  target_link_libraries(simdjson-fuzzer INTERFACE ${SIMDJSON_FUZZ_LDFLAGS})
 
   # Define the fuzzers
   add_custom_target(all_fuzzers)

fuzz/build_fuzzer_variants.sh

@@ -23,9 +23,32 @@ fi
 # detect unset variables
 set -u
 
+if ! which clang++$CLANGSUFFIX >/dev/null 2>&1 ; then
+  echo "could not find clang++$CLANGSUFFIX"
+  exit 1
+fi
+
+# find out how to build fuzzer. On amd64 and arm64, libFuzzer is built with the compiler and activated
+# with -fsanitize=fuzzer at link time. On power, libFuzzer is shipped separately.
+testfuzzer=testfuzzer.cpp
+/bin/echo -e "#include <cstddef>\n#include <cstdint>\nextern \"C\" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {return 0;}" >$testfuzzer
+if clang++$CLANGSUFFIX -o testfuzzer $testfuzzer -fsanitize=fuzzer && ./testfuzzer -runs=1 >/dev/null 2>&1 ; then
+   echo "will use -fsanitize=fuzzer to link libFuzzer"
+   SIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer"
+elif clang++$CLANGSUFFIX -o testfuzzer $testfuzzer -fsanitize=fuzzer-no-link -lFuzzer  && ./testfuzzer -runs=1 >/dev/null 2>&1 ; then
+   echo "will use -lFuzzer to link libFuzzer"
+   SIMDJSON_FUZZ_LDFLAGS="-lFuzzer"
+else
+  echo "could not link to the fuzzer with -fsanitize=fuzzer or -lFuzzer"
+  exit 1
+fi
+
+if [ -e testfuzzer ] ; then rm testfuzzer; fi
+if [ -e $testfuzzer ] ; then rm $testfuzzer; fi
+
 # common options
 CXX_CLAGS_COMMON=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
-COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_BUILD_STATIC=Off -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On"
+COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_BUILD_STATIC=Off -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On -DSIMDJSON_FUZZ_LDFLAGS=$SIMDJSON_FUZZ_LDFLAGS"
 
 # A replay build, as plain as it gets. For use with valgrind/gdb.
 variant=replay
@@ -67,8 +90,7 @@ variant=sanitizers-O3
 	      -DCMAKE_CXX_FLAGS="-O3 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined $CXX_CLAGS_COMMON" \
 	      -DCMAKE_C_FLAGS="-O3 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \
 	      -DCMAKE_BUILD_TYPE=Debug \
-	      -DSIMDJSON_FUZZ_LINKMAIN=Off \
+	      -DSIMDJSON_FUZZ_LINKMAIN=Off
-	      -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer"
 
 	ninja all_fuzzers
 	cd ..
@@ -85,8 +107,7 @@ variant=sanitizers-O0
 	      -DCMAKE_CXX_FLAGS="-O0 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined $CXX_CLAGS_COMMON" \
 	      -DCMAKE_C_FLAGS="-O0 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \
 	      -DCMAKE_BUILD_TYPE=Debug \
-	      -DSIMDJSON_FUZZ_LINKMAIN=Off \
+	      -DSIMDJSON_FUZZ_LINKMAIN=Off
-	      -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer"
 
 	ninja all_fuzzers
 	cd ..
@@ -105,8 +126,7 @@ variant=fast
 	      -DCMAKE_CXX_FLAGS="-fsanitize=fuzzer-no-link $CXX_CLAGS_COMMON" \
 	      -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link" \
 	      -DCMAKE_BUILD_TYPE=Release \
-	      -DSIMDJSON_FUZZ_LINKMAIN=Off \
+	      -DSIMDJSON_FUZZ_LINKMAIN=Off
-	      -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer"
 
 	ninja all_fuzzers
 	cd ..