diff --git a/.circleci/config.yml b/.circleci/config.yml index 43fcfe14..02a0af4f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,7 @@ jobs: sanitize-gcc9: description: Build and run tests on GCC 9 and AVX 2 with a cmake sanitize build executor: gcc9 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: -j4 --output-on-failure -E checkperf } + environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: -j4 --output-on-failure -E checkperf } steps: [ install_cmake, cmake_test_all ] sanitize-clang6: description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build diff --git a/CMakeLists.txt b/CMakeLists.txt index cb45a119..53022256 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -182,12 +182,7 @@ add_subdirectory(dependencies) add_subdirectory(tests) add_subdirectory(examples) add_subdirectory(benchmark) - -# for fuzzing, read the comments in the fuzz/CMakeLists.txt file -option(ENABLE_FUZZING "enable building the fuzzers" ON) -if(ENABLE_FUZZING AND SIMDJSON_EXCEPTIONS) - add_subdirectory(fuzz) -endif() +add_subdirectory(fuzz) # # CPack diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index d0606ec7..e0efa486 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -1,56 +1,59 @@ -# First attempt at a fuzzer, using libFuzzer. -# -# compile like this: -# mkdir build-fuzzer -# cd build-fuzzer -# export LDFLAGS="-fsanitize=address,undefined" -# export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined" -# export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined" -# export CXX=clang++ -# export CC=clang++ -# cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer -# ninja +option(ENABLE_FUZZING "enable building the fuzzers" ON) +if(ENABLE_FUZZING) -# settings this links in a main. useful for reproducing, -# kcov, gdb, afl, valgrind. -# (note that libFuzzer can also reproduce, just pass it the files) -# -# Using this by default, means the fuzzers will be built as a part of the normal -# workflow, meaning they wont bitrot and will participate in refactoring etc. -# -option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On) + # First attempt at a fuzzer, using libFuzzer. + # + # compile like this: + # mkdir build-fuzzer + # cd build-fuzzer + # export LDFLAGS="-fsanitize=address,undefined" + # export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined" + # export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined" + # export CXX=clang++ + # export CC=clang++ + # cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer + # ninja -# For oss-fuzz - insert $LIB_FUZZING_ENGINE into the link flags, but only for -# the fuzz targets, otherwise the cmake configuration step fails. -set(SIMDJSON_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets") + # settings this links in a main. useful for reproducing, + # kcov, gdb, afl, valgrind. + # (note that libFuzzer can also reproduce, just pass it the files) + # + # Using this by default, means the fuzzers will be built as a part of the normal + # workflow, meaning they wont bitrot and will participate in refactoring etc. + # + option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On) -set(SOURCES - fuzz_parser.cpp - fuzz_minify.cpp - fuzz_dump.cpp - fuzz_print_json.cpp - fuzz_dump_raw_tape.cpp -) + # For oss-fuzz - insert $LIB_FUZZING_ENGINE into the link flags, but only for + # the fuzz targets, otherwise the cmake configuration step fails. + set(SIMDJSON_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets") -add_custom_target(print_all_fuzz_targets + add_custom_target(print_all_fuzz_targets COMMAND ${CMAKE_COMMAND} -E echo ${SOURCES} -) + ) -macro(implement_fuzzer sourcefile) - get_filename_component(basename ${sourcefile} NAME_WE) - set(name ${basename}) - add_executable(${name} ${sourcefile}) + # Fuzzer build flags and libraries + add_library(simdjson-fuzzer INTERFACE) if (SIMDJSON_FUZZ_LINKMAIN) - target_sources(${name} PRIVATE main.cpp) + target_link_libraries(simdjson-fuzzer INTERFACE simdjson-source) + target_sources(simdjson-fuzzer INTERFACE $/main.cpp) + else () + target_link_libraries(simdjson-fuzzer INTERFACE simdjson) endif () - target_link_libraries(${name} PRIVATE simdjson simdjson-flags) -if (SIMDJSON_FUZZ_LDFLAGS) - target_link_libraries(${name} PRIVATE ${SIMDJSON_FUZZ_LDFLAGS}) -endif () -endmacro () + target_link_libraries(simdjson-fuzzer INTERFACE simdjson-flags) + target_link_libraries(simdjson-fuzzer INTERFACE ${SIMDJSON_FUZZ_LDFLAGS}) + + function(implement_fuzzer name) + add_executable(${name} ${name}.cpp) + target_link_libraries(${name} PRIVATE simdjson-fuzzer) + add_test(${name} ${name}) + set_property(TEST ${name} APPEND PROPERTY LABELS fuzz) + endfunction() -foreach (X IN ITEMS ${SOURCES}) - implement_fuzzer(${X}) -endforeach () + implement_fuzzer(fuzz_parser) + implement_fuzzer(fuzz_minify) + implement_fuzzer(fuzz_dump) + implement_fuzzer(fuzz_print_json) + implement_fuzzer(fuzz_dump_raw_tape) +endif() diff --git a/fuzz/fuzz_dump.cpp b/fuzz/fuzz_dump.cpp index 868a7967..fc509182 100644 --- a/fuzz/fuzz_dump.cpp +++ b/fuzz/fuzz_dump.cpp @@ -9,57 +9,53 @@ // example from doc/basics.md#tree-walking-and-json-element-types static void print_json(std::ostream& os, simdjson::dom::element element) { -#if SIMDJSON_EXCEPTIONS - const char endl='\n'; - switch (element.type()) { - case simdjson::dom::element_type::ARRAY: - os << "["; - for (simdjson::dom::element child : simdjson::dom::array(element)) { - print_json(os, child); - os << ","; - } - os << "]"; - break; - case simdjson::dom::element_type::OBJECT: - os << "{"; - for (simdjson::dom::key_value_pair field : simdjson::dom::object(element)) { - os << "\"" << field.key << "\": "; - print_json(os, field.value); - } - os << "}"; - break; - case simdjson::dom::element_type::INT64: - os << int64_t(element) << endl; - break; - case simdjson::dom::element_type::UINT64: - os << uint64_t(element) << endl; - break; - case simdjson::dom::element_type::DOUBLE: - os << double(element) << endl; - break; - case simdjson::dom::element_type::STRING: - os << std::string_view(element) << endl; - break; - case simdjson::dom::element_type::BOOL: - os << bool(element) << endl; - break; - case simdjson::dom::element_type::NULL_VALUE: - os << "null" << endl; - break; + const char endl='\n'; + switch (element.type()) { + case simdjson::dom::element_type::ARRAY: + os << "["; + for (simdjson::dom::element child : element.get().first) { + print_json(os, child); + os << ","; } -#endif + os << "]"; + break; + case simdjson::dom::element_type::OBJECT: + os << "{"; + for (simdjson::dom::key_value_pair field : element.get().first) { + os << "\"" << field.key << "\": "; + print_json(os, field.value); + } + os << "}"; + break; + case simdjson::dom::element_type::INT64: + os << element.get().first << endl; + break; + case simdjson::dom::element_type::UINT64: + os << element.get().first << endl; + break; + case simdjson::dom::element_type::DOUBLE: + os << element.get().first << endl; + break; + case simdjson::dom::element_type::STRING: + os << element.get().first << endl; + break; + case simdjson::dom::element_type::BOOL: + os << element.get().first << endl; + break; + case simdjson::dom::element_type::NULL_VALUE: + os << "null" << endl; + break; + } } extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { -#if SIMDJSON_EXCEPTIONS - try { - simdjson::dom::parser pj; - auto elem=pj.parse(Data, Size); - auto v=elem.value(); - NulOStream os; - //std::ostream& os(std::cout); - print_json(os,v); - } catch (...) { - } -#endif - return 0; + simdjson::dom::parser parser; + simdjson::error_code error; + simdjson::dom::element elem; + parser.parse(Data, Size).tie(elem, error); + + if (error) { return 1; } + NulOStream os; + //std::ostream& os(std::cout); + print_json(os,elem); + return 0; } diff --git a/fuzz/fuzz_dump_raw_tape.cpp b/fuzz/fuzz_dump_raw_tape.cpp index 5de7654a..c4343ede 100644 --- a/fuzz/fuzz_dump_raw_tape.cpp +++ b/fuzz/fuzz_dump_raw_tape.cpp @@ -7,15 +7,13 @@ #include "NullBuffer.h" extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { -#if SIMDJSON_EXCEPTIONS - try { - simdjson::dom::parser pj; - auto elem=pj.parse(Data, Size); - auto v=elem.value(); - NulOStream os; - UNUSED auto dumpstatus=v.dump_raw_tape(os); - } catch (...) { - } -#endif + simdjson::dom::parser parser; + simdjson::error_code error; + simdjson::dom::element elem; + parser.parse(Data, Size).tie(elem, error); + if (error) { return 1; } + + NulOStream os; + UNUSED auto dumpstatus = elem.dump_raw_tape(os); return 0; } diff --git a/fuzz/fuzz_minify.cpp b/fuzz/fuzz_minify.cpp index 19223a94..a8425a0a 100644 --- a/fuzz/fuzz_minify.cpp +++ b/fuzz/fuzz_minify.cpp @@ -8,15 +8,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { auto end = begin + Size; std::string str(begin, end); -#if SIMDJSON_EXCEPTIONS - try { - simdjson::dom::parser parser; - simdjson::dom::element doc = parser.parse(str); - std::string minified=simdjson::minify(doc); - (void)minified; - } catch (...) { + simdjson::dom::parser parser; + simdjson::error_code error; + simdjson::dom::element elem; + parser.parse(str).tie(elem, error); + if (error) { return 1; } - } -#endif + std::string minified=simdjson::minify(elem); + (void)minified; return 0; } diff --git a/fuzz/fuzz_parser.cpp b/fuzz/fuzz_parser.cpp index 25a880d8..610cead4 100644 --- a/fuzz/fuzz_parser.cpp +++ b/fuzz/fuzz_parser.cpp @@ -3,13 +3,9 @@ #include #include extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { -#if SIMDJSON_EXCEPTIONS - try { - simdjson::dom::parser pj; - auto result=pj.parse(Data, Size); - UNUSED auto v=result.value(); - }catch(...) { - } -#endif - return 0; + simdjson::dom::parser parser; + UNUSED simdjson::error_code error; + UNUSED simdjson::dom::element elem; + parser.parse(Data, Size).tie(elem, error); + return 0; } diff --git a/fuzz/fuzz_print_json.cpp b/fuzz/fuzz_print_json.cpp index 9811f155..02db2941 100644 --- a/fuzz/fuzz_print_json.cpp +++ b/fuzz/fuzz_print_json.cpp @@ -7,14 +7,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { -#if SIMDJSON_EXCEPTIONS - try { - simdjson::dom::parser pj; - auto elem=pj.parse(Data, Size); - NulOStream os; - os<