From d539781cf3f1a2f8392e2cbbdbb241b60ecb6413 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Fri, 7 May 2021 22:59:26 -0400 Subject: [PATCH] This attempts to fix the fuzzers. (#1564) * This attempts to fix the fuzzers. * Retiring bintray. * Disabling ARM fuzzing. --- .appveyor.yml | 8 +-- .circleci/config.yml | 26 ++++----- .drone.yml | 72 ++++++++++++------------ .github/workflows/mingw-ci.yml | 4 +- .github/workflows/mingw64-ci.yml | 6 +- .github/workflows/msys2.yml | 2 +- .github/workflows/power-fuzz.yml | 17 ++++-- .github/workflows/ubuntu18-checkperf.yml | 2 +- .github/workflows/ubuntu18.yml | 2 +- .github/workflows/ubuntu20-checkperf.yml | 2 +- .github/workflows/ubuntu20-noexcept.yml | 2 +- .github/workflows/ubuntu20-nothread.yml | 2 +- .github/workflows/ubuntu20.yml | 2 +- .github/workflows/vs16-clang-ci.yml | 2 +- .github/workflows/vs16-ninja-ci.yml | 2 +- .travis.yml | 2 +- fuzz/CMakeLists.txt | 8 +-- fuzz/build_fuzzer_variants.sh | 2 +- fuzz/measure_coverage.sh | 5 +- fuzz/ossfuzz.sh | 5 +- 20 files changed, 93 insertions(+), 80 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 7f016e85..2d12fe3a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,19 +18,19 @@ environment: CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM? - job_name: VS2017 (Static, No Threads) image: Visual Studio 2017 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32) platform: Win32 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32, No Exceptions) platform: Win32 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF CTEST_ARGS: -LE explicitonly - job_name: VS2015 image: Visual Studio 2015 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly build_script: diff --git a/.circleci/config.yml b/.circleci/config.yml index 415a5911..6aa0749f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -164,60 +164,60 @@ jobs: gcc10-perftest: description: Build and run performance tests on GCC 10 and AVX 2 with a cmake static build, this test performance regression executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_perftest ] gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake static build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ] clang6: description: Build and run tests on clang 6 and AVX 2 with a cmake static build executor: clang6 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test ] clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake static build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ] # libcpp libcpp-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake static build and libc++ executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ] # sanitize sanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] threadsanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] threadsanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] # dynamic dynamic-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON } steps: [ cmake_test, cmake_install_test ] dynamic-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake dynamic build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON } steps: [ cmake_test, cmake_install_test ] # unthreaded @@ -262,12 +262,12 @@ jobs: sanitize-haswell-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-haswell-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] workflows: diff --git a/.drone.yml b/.drone.yml index eafa52cb..1412d392 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,7 +8,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')" @@ -29,7 +29,7 @@ steps: CC: clang-6.0 CXX: clang++-6.0 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')" @@ -50,7 +50,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -77,7 +77,7 @@ steps: CC: clang-6.0 CXX: clang++-6.0 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - mkdir build @@ -100,7 +100,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -122,7 +122,7 @@ steps: environment: CC: clang-9 CXX: clang++-9 - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON BUILD_FLAGS: -- -j CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: @@ -142,7 +142,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -213,7 +213,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -237,7 +237,7 @@ steps: environment: CC: clang-6.0 CXX: clang++-6.0 - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON BUILD_FLAGS: -- -j CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: @@ -260,7 +260,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -281,7 +281,7 @@ steps: environment: CC: clang-6.0 CXX: clang++-6.0 - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON BUILD_FLAGS: -- -j CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: @@ -302,7 +302,7 @@ steps: image: gcc:8 environment: BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CC: gcc CXX: g++ @@ -356,7 +356,7 @@ steps: CC: clang-9 CXX: clang++-9 BUILD_FLAGS: -- -j 4 - CMAKE_FLAGS: -GNinja -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -GNinja -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CXXFLAGS: -stdlib=libc++ commands: @@ -377,7 +377,7 @@ steps: CC: clang-9 CXX: clang++-9 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CXXFLAGS: -stdlib=libc++ commands: @@ -398,7 +398,7 @@ steps: CC: clang-7 CXX: clang++-7 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CXXFLAGS: -stdlib=libc++ commands: @@ -429,26 +429,28 @@ steps: - cmake $CMAKE_FLAGS .. - cmake --build . $BUILD_FLAGS - ctest $CTEST_FLAGS ---- -kind: pipeline -name: arm64-fuzz -platform: { os: linux, arch: arm64 } -steps: -- name: Build and run fuzzers shortly - image: ubuntu:20.04 - environment: - CC: clang - CXX: clang++ - DEBIAN_FRONTEND: noninteractive - ASAN_OPTIONS: detect_leaks=0 - commands: - - apt-get -qq update - - apt-get install -q -y clang cmake git wget zip ninja-build - - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar - - tar xf corpus.tar && rm corpus.tar - - fuzz/build_like_ossfuzz.sh - - mkdir -p common_out - - for fuzzer in build/fuzz/fuzz_* ; do echo $fuzzer;$fuzzer common_out out/* -max_total_time=40; done +############################## +# With bintray gone, it is clear how we should fuzz. +#--- +#kind: pipeline +#name: arm64-fuzz +#platform: { os: linux, arch: arm64 } +#steps: +#- name: Build and run fuzzers shortly +# image: ubuntu:20.04 +# environment: +# CC: clang +# CXX: clang++ +# DEBIAN_FRONTEND: noninteractive +# ASAN_OPTIONS: detect_leaks=0 +# commands: +# - apt-get -qq update +# - apt-get install -q -y clang cmake git wget zip ninja-build +# - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar +# - tar xf corpus.tar && rm corpus.tar +# - fuzz/build_like_ossfuzz.sh +# - mkdir -p common_out +# - for fuzzer in build/fuzz/fuzz_* ; do echo $fuzzer;$fuzzer common_out out/* -max_total_time=40; done --- kind: pipeline name: stylecheck diff --git a/.github/workflows/mingw-ci.yml b/.github/workflows/mingw-ci.yml index 74a9cb8b..273c3880 100644 --- a/.github/workflows/mingw-ci.yml +++ b/.github/workflows/mingw-ci.yml @@ -22,7 +22,7 @@ jobs: runs-on: windows-2016 env: - CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line + CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line CC: gcc CXX: g++ @@ -63,6 +63,6 @@ jobs: git --version mkdir build32 cd build32 - cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. cmake --build . --target acceptance_tests --verbose ctest -L acceptance --output-on-failure diff --git a/.github/workflows/mingw64-ci.yml b/.github/workflows/mingw64-ci.yml index b4281356..c032b8fa 100644 --- a/.github/workflows/mingw64-ci.yml +++ b/.github/workflows/mingw64-ci.yml @@ -22,7 +22,7 @@ jobs: runs-on: windows-2016 env: - CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line + CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line CC: gcc CXX: g++ @@ -63,12 +63,12 @@ jobs: git --version mkdir build64 cd build64 - cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. cmake --build . --target acceptance_tests --verbose ctest -L acceptance --output-on-failure cd .. mkdir build64debug cd build64debug - cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. cmake --build . --target acceptance_tests --verbose ctest -L acceptance --output-on-failure diff --git a/.github/workflows/msys2.yml b/.github/workflows/msys2.yml index 734103dc..d27d8bce 100644 --- a/.github/workflows/msys2.yml +++ b/.github/workflows/msys2.yml @@ -52,6 +52,6 @@ jobs: run: | mkdir build cd build - cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON .. + cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON .. cmake --build . --verbose ctest -j4 --output-on-failure -LE explicitonly diff --git a/.github/workflows/power-fuzz.yml b/.github/workflows/power-fuzz.yml index 356f2b51..77a98841 100644 --- a/.github/workflows/power-fuzz.yml +++ b/.github/workflows/power-fuzz.yml @@ -16,6 +16,16 @@ jobs: name: Build on ubuntu-20.04 ppc64le steps: - uses: actions/checkout@v2.1.0 + - uses: actions/cache@v2 + id: cache-corpus + with: + path: out/ + key: corpus-${{ github.run_id }} + restore-keys: corpus- + - name: show statistics for the cached corpus + run: | + echo number of files in github action corpus cache: + find out -type f |wc -l - uses: uraimo/run-on-arch-action@v2.0.5 name: Run commands id: runcmd @@ -42,8 +52,9 @@ jobs: cmake .. -GNinja \ -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX \ -DCMAKE_C_COMPILER=clang$CLANGSUFFIX \ - -DSIMDJSON_BUILD_STATIC=Off \ - -DENABLE_FUZZING=On \ + -DBUILD_SHARED_LIBS=OFF \ + -DSIMDJSON_DEVELOPER_MODE=ON \ + -DSIMDJSON_ENABLE_FUZZING=On \ -DSIMDJSON_COMPETITION=OFF \ -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ -DSIMDJSON_DISABLE_DEPRECATED_API=On \ @@ -55,8 +66,6 @@ jobs: cd .. builddir=build cmake --build $builddir - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar - tar xf corpus.tar && rm corpus.tar fuzzernames=$(cmake --build $builddir --target print_all_fuzzernames |tail -n1) for fuzzer in $fuzzernames ; do exe=$builddir/fuzz/$fuzzer diff --git a/.github/workflows/ubuntu18-checkperf.yml b/.github/workflows/ubuntu18-checkperf.yml index 29254230..7836065a 100644 --- a/.github/workflows/ubuntu18-checkperf.yml +++ b/.github/workflows/ubuntu18-checkperf.yml @@ -24,6 +24,6 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . --target checkperf && ctest --output-on-failure -R checkperf ubuntu18-checkperf.yml diff --git a/.github/workflows/ubuntu18.yml b/.github/workflows/ubuntu18.yml index 6436c26d..bfbda67d 100644 --- a/.github/workflows/ubuntu18.yml +++ b/.github/workflows/ubuntu18.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && make install && diff --git a/.github/workflows/ubuntu20-checkperf.yml b/.github/workflows/ubuntu20-checkperf.yml index fcea860d..bb17ac03 100644 --- a/.github/workflows/ubuntu20-checkperf.yml +++ b/.github/workflows/ubuntu20-checkperf.yml @@ -24,6 +24,6 @@ jobs: run: | mkdir build && cd build && - cmake -DCMAKE_CXX_FLAGS="-Werror=old-style-cast -pedantic -Wpedantic" -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DCMAKE_CXX_FLAGS="-Werror=old-style-cast -pedantic -Wpedantic" -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . --target checkperf && ctest --output-on-failure -R checkperf diff --git a/.github/workflows/ubuntu20-noexcept.yml b/.github/workflows/ubuntu20-noexcept.yml index 789e1c49..1eaafbfd 100644 --- a/.github/workflows/ubuntu20-noexcept.yml +++ b/.github/workflows/ubuntu20-noexcept.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_EXCEPTIONS=OFF -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_EXCEPTIONS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && make install && diff --git a/.github/workflows/ubuntu20-nothread.yml b/.github/workflows/ubuntu20-nothread.yml index 2af570e6..60a9614b 100644 --- a/.github/workflows/ubuntu20-nothread.yml +++ b/.github/workflows/ubuntu20-nothread.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_ENABLE_THREADS=OFF -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_ENABLE_THREADS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && make install && diff --git a/.github/workflows/ubuntu20.yml b/.github/workflows/ubuntu20.yml index 1fcde2e6..53d61358 100644 --- a/.github/workflows/ubuntu20.yml +++ b/.github/workflows/ubuntu20.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && cmake --install . && diff --git a/.github/workflows/vs16-clang-ci.yml b/.github/workflows/vs16-clang-ci.yml index 3940caaf..6068d7ab 100644 --- a/.github/workflows/vs16-clang-ci.yml +++ b/.github/workflows/vs16-clang-ci.yml @@ -29,7 +29,7 @@ jobs: buildDirectory: "${{ github.workspace }}/../../_temp/windows" cmakeBuildType: Release buildWithCMake: true - cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=ON + cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=OFF buildWithCMakeArgs: --config Release - name: 'Run CTest' diff --git a/.github/workflows/vs16-ninja-ci.yml b/.github/workflows/vs16-ninja-ci.yml index 7b40bf2b..2b36fdf4 100644 --- a/.github/workflows/vs16-ninja-ci.yml +++ b/.github/workflows/vs16-ninja-ci.yml @@ -29,7 +29,7 @@ jobs: buildDirectory: "${{ github.workspace }}/../../_temp/windows" cmakeBuildType: Release buildWithCMake: true - cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=ON + cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=OFF buildWithCMakeArgs: --config Release - name: 'Run CTest' run: ctest -C Release -LE explicitonly --output-on-failure diff --git a/.travis.yml b/.travis.yml index c436c5fc..276d30b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -176,7 +176,7 @@ install: export ASAN_OPTIONS="detect_leaks=0"; fi - if [[ "${STATIC}" == "on" ]]; then - export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_BUILD_STATIC=ON"; + export CMAKE_FLAGS="${CMAKE_FLAGS} -DBUILD_SHARED_LIBS=OFF"; fi - export CTEST_FLAGS="-j4 --output-on-failure -LE explicitonly" diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 5d36fd3a..e903e07a 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -1,11 +1,11 @@ if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL) - option(ENABLE_FUZZING "enable building the fuzzers" ON) + option(SIMDJSON_ENABLE_FUZZING "enable building the fuzzers" ON) else() - option(ENABLE_FUZZING "enable building the fuzzers" OFF) + option(SIMDJSON_ENABLE_FUZZING "enable building the fuzzers" OFF) endif() -if(ENABLE_FUZZING) +if(SIMDJSON_ENABLE_FUZZING) # First attempt at a fuzzer, using libFuzzer. # @@ -17,7 +17,7 @@ if(ENABLE_FUZZING) # export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined" # export CXX=clang++ # export CC=clang++ - # cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer + # cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_ENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer # ninja # settings this links in a main. useful for reproducing, diff --git a/fuzz/build_fuzzer_variants.sh b/fuzz/build_fuzzer_variants.sh index 6a532df1..7aa9e87e 100755 --- a/fuzz/build_fuzzer_variants.sh +++ b/fuzz/build_fuzzer_variants.sh @@ -48,7 +48,7 @@ if [ -e $testfuzzer ] ; then rm $testfuzzer; fi # common options CXX_CLAGS_COMMON=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_BUILD_STATIC=Off -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On -DSIMDJSON_FUZZ_LDFLAGS=$SIMDJSON_FUZZ_LDFLAGS" +COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On -DSIMDJSON_FUZZ_LDFLAGS=$SIMDJSON_FUZZ_LDFLAGS" # A replay build, as plain as it gets. For use with valgrind/gdb. variant=replay diff --git a/fuzz/measure_coverage.sh b/fuzz/measure_coverage.sh index 33dbfc94..480087ec 100755 --- a/fuzz/measure_coverage.sh +++ b/fuzz/measure_coverage.sh @@ -23,8 +23,9 @@ if [ ! -d $bdir ] ; then cmake .. \ -GNinja \ -DCMAKE_BUILD_TYPE=Debug \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ + -DSIMDJSON_DEVELOPER_MODE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DSIMDJSON_ENABLE_FUZZING=On \ -DSIMDJSON_DISABLE_DEPRECATED_API=On \ -DSIMDJSON_FUZZ_LINKMAIN=On ninja all_fuzzers diff --git a/fuzz/ossfuzz.sh b/fuzz/ossfuzz.sh index b4cf9e7b..18b68027 100755 --- a/fuzz/ossfuzz.sh +++ b/fuzz/ossfuzz.sh @@ -26,8 +26,9 @@ cd build cmake .. \ -GNinja \ -DCMAKE_BUILD_TYPE=Debug \ --DSIMDJSON_BUILD_STATIC=On \ --DENABLE_FUZZING=On \ +-DSIMDJSON_DEVELOPER_MODE=ON \ +-DBUILD_SHARED_LIBS=OFF \ +-DSIMDJSON_ENABLE_FUZZING=On \ -DSIMDJSON_COMPETITION=Off \ -DSIMDJSON_FUZZ_LINKMAIN=Off \ -DSIMDJSON_GOOGLE_BENCHMARKS=Off \