jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Merge pull request #728 from simdjson/jkeiser/cmake-fuzz-noexceptions 

Compile fuzzers without exceptions, run as part of tests
This commit is contained in:
John Keiser 2020-04-23 10:13:33 -07:00 committed by GitHub
parent 382392e03b 7a2fda891c
commit 587ba9bec0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 125 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/config.yml
View File

@ -118,7 +118,7 @@ jobs:
sanitize-gcc9: sanitize-gcc9:
description: Build and run tests on GCC 9 and AVX 2 with a cmake sanitize build description: Build and run tests on GCC 9 and AVX 2 with a cmake sanitize build
executor: gcc9 executor: gcc9
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: -j4 --output-on-failure -E checkperf } environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
steps: [ install_cmake, cmake_test_all ] steps: [ install_cmake, cmake_test_all ]
sanitize-clang6: sanitize-clang6:
description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build

7
CMakeLists.txt
View File

@ -182,12 +182,7 @@ add_subdirectory(dependencies)
add_subdirectory(tests) add_subdirectory(tests)
add_subdirectory(examples) add_subdirectory(examples)
add_subdirectory(benchmark) add_subdirectory(benchmark)
add_subdirectory(fuzz)
# for fuzzing, read the comments in the fuzz/CMakeLists.txt file
option(ENABLE_FUZZING "enable building the fuzzers" ON)
if(ENABLE_FUZZING AND SIMDJSON_EXCEPTIONS)
add_subdirectory(fuzz)
endif()
# #
# CPack # CPack

93
fuzz/CMakeLists.txt
View File

@ -1,56 +1,59 @@
# First attempt at a fuzzer, using libFuzzer. option(ENABLE_FUZZING "enable building the fuzzers" ON)
#
# compile like this:
# mkdir build-fuzzer
# cd build-fuzzer
# export LDFLAGS="-fsanitize=address,undefined"
# export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CXX=clang++
# export CC=clang++
# cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer
# ninja
if(ENABLE_FUZZING)
# settings this links in a main. useful for reproducing, # First attempt at a fuzzer, using libFuzzer.
# kcov, gdb, afl, valgrind. #
# (note that libFuzzer can also reproduce, just pass it the files) # compile like this:
# # mkdir build-fuzzer
# Using this by default, means the fuzzers will be built as a part of the normal # cd build-fuzzer
# workflow, meaning they wont bitrot and will participate in refactoring etc. # export LDFLAGS="-fsanitize=address,undefined"
# # export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On) # export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CXX=clang++
# export CC=clang++
# cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer
# ninja
# For oss-fuzz - insert $LIB_FUZZING_ENGINE into the link flags, but only for # settings this links in a main. useful for reproducing,
# the fuzz targets, otherwise the cmake configuration step fails. # kcov, gdb, afl, valgrind.
set(SIMDJSON_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets") # (note that libFuzzer can also reproduce, just pass it the files)
#
# Using this by default, means the fuzzers will be built as a part of the normal
# workflow, meaning they wont bitrot and will participate in refactoring etc.
#
option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On)
set(SOURCES # For oss-fuzz - insert $LIB_FUZZING_ENGINE into the link flags, but only for
fuzz_parser.cpp # the fuzz targets, otherwise the cmake configuration step fails.
fuzz_minify.cpp set(SIMDJSON_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets")
fuzz_dump.cpp
fuzz_print_json.cpp
fuzz_dump_raw_tape.cpp
)
add_custom_target(print_all_fuzz_targets add_custom_target(print_all_fuzz_targets
COMMAND ${CMAKE_COMMAND} -E echo ${SOURCES} COMMAND ${CMAKE_COMMAND} -E echo ${SOURCES}
) )
macro(implement_fuzzer sourcefile) # Fuzzer build flags and libraries
get_filename_component(basename ${sourcefile} NAME_WE) add_library(simdjson-fuzzer INTERFACE)
set(name ${basename})
add_executable(${name} ${sourcefile})
if (SIMDJSON_FUZZ_LINKMAIN) if (SIMDJSON_FUZZ_LINKMAIN)
target_sources(${name} PRIVATE main.cpp) target_link_libraries(simdjson-fuzzer INTERFACE simdjson-source)
target_sources(simdjson-fuzzer INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/main.cpp)
else ()
target_link_libraries(simdjson-fuzzer INTERFACE simdjson)
endif () endif ()
target_link_libraries(${name} PRIVATE simdjson simdjson-flags) target_link_libraries(simdjson-fuzzer INTERFACE simdjson-flags)
if (SIMDJSON_FUZZ_LDFLAGS) target_link_libraries(simdjson-fuzzer INTERFACE ${SIMDJSON_FUZZ_LDFLAGS})
target_link_libraries(${name} PRIVATE ${SIMDJSON_FUZZ_LDFLAGS})
endif () function(implement_fuzzer name)
endmacro () add_executable(${name} ${name}.cpp)
target_link_libraries(${name} PRIVATE simdjson-fuzzer)
add_test(${name} ${name})
set_property(TEST ${name} APPEND PROPERTY LABELS fuzz)
endfunction()
foreach (X IN ITEMS ${SOURCES}) implement_fuzzer(fuzz_parser)
implement_fuzzer(${X}) implement_fuzzer(fuzz_minify)
endforeach () implement_fuzzer(fuzz_dump)
implement_fuzzer(fuzz_print_json)
implement_fuzzer(fuzz_dump_raw_tape)
endif()

96
fuzz/fuzz_dump.cpp
View File

@ -9,57 +9,53 @@
// example from doc/basics.md#tree-walking-and-json-element-types // example from doc/basics.md#tree-walking-and-json-element-types
static void print_json(std::ostream& os, simdjson::dom::element element) { static void print_json(std::ostream& os, simdjson::dom::element element) {
#if SIMDJSON_EXCEPTIONS const char endl='\n';
const char endl='\n'; switch (element.type()) {
switch (element.type()) { case simdjson::dom::element_type::ARRAY:
case simdjson::dom::element_type::ARRAY: os << "[";
os << "["; for (simdjson::dom::element child : element.get<simdjson::dom::array>().first) {
for (simdjson::dom::element child : simdjson::dom::array(element)) { print_json(os, child);
print_json(os, child); os << ",";
os << ",";
}
os << "]";
break;
case simdjson::dom::element_type::OBJECT:
os << "{";
for (simdjson::dom::key_value_pair field : simdjson::dom::object(element)) {
os << "\"" << field.key << "\": ";
print_json(os, field.value);
}
os << "}";
break;
case simdjson::dom::element_type::INT64:
os << int64_t(element) << endl;
break;
case simdjson::dom::element_type::UINT64:
os << uint64_t(element) << endl;
break;
case simdjson::dom::element_type::DOUBLE:
os << double(element) << endl;
break;
case simdjson::dom::element_type::STRING:
os << std::string_view(element) << endl;
break;
case simdjson::dom::element_type::BOOL:
os << bool(element) << endl;
break;
case simdjson::dom::element_type::NULL_VALUE:
os << "null" << endl;
break;
} }
#endif os << "]";
break;
case simdjson::dom::element_type::OBJECT:
os << "{";
for (simdjson::dom::key_value_pair field : element.get<simdjson::dom::object>().first) {
os << "\"" << field.key << "\": ";
print_json(os, field.value);
}
os << "}";
break;
case simdjson::dom::element_type::INT64:
os << element.get<int64_t>().first << endl;
break;
case simdjson::dom::element_type::UINT64:
os << element.get<uint64_t>().first << endl;
break;
case simdjson::dom::element_type::DOUBLE:
os << element.get<double>().first << endl;
break;
case simdjson::dom::element_type::STRING:
os << element.get<std::string_view>().first << endl;
break;
case simdjson::dom::element_type::BOOL:
os << element.get<bool>().first << endl;
break;
case simdjson::dom::element_type::NULL_VALUE:
os << "null" << endl;
break;
}
} }
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS simdjson::dom::parser parser;
try { simdjson::error_code error;
simdjson::dom::parser pj; simdjson::dom::element elem;
auto elem=pj.parse(Data, Size); parser.parse(Data, Size).tie(elem, error);
auto v=elem.value();
NulOStream os; if (error) { return 1; }
//std::ostream& os(std::cout); NulOStream os;
print_json(os,v); //std::ostream& os(std::cout);
} catch (...) { print_json(os,elem);
} return 0;
#endif
return 0;
} }

18
fuzz/fuzz_dump_raw_tape.cpp
View File

@ -7,15 +7,13 @@
#include "NullBuffer.h" #include "NullBuffer.h"
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS simdjson::dom::parser parser;
try { simdjson::error_code error;
simdjson::dom::parser pj; simdjson::dom::element elem;
auto elem=pj.parse(Data, Size); parser.parse(Data, Size).tie(elem, error);
auto v=elem.value(); if (error) { return 1; }
NulOStream os;
UNUSED auto dumpstatus=v.dump_raw_tape(os); NulOStream os;
} catch (...) { UNUSED auto dumpstatus = elem.dump_raw_tape(os);
}
#endif
return 0; return 0;
} }

16
fuzz/fuzz_minify.cpp
View File

@ -8,15 +8,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
auto end = begin + Size; auto end = begin + Size;
std::string str(begin, end); std::string str(begin, end);
#if SIMDJSON_EXCEPTIONS simdjson::dom::parser parser;
try { simdjson::error_code error;
simdjson::dom::parser parser; simdjson::dom::element elem;
simdjson::dom::element doc = parser.parse(str); parser.parse(str).tie(elem, error);
std::string minified=simdjson::minify(doc); if (error) { return 1; }
(void)minified;
} catch (...) {
} std::string minified=simdjson::minify(elem);
#endif (void)minified;
return 0; return 0;
} }

14
fuzz/fuzz_parser.cpp
View File

@ -3,13 +3,9 @@
#include <cstdint> #include <cstdint>
#include <string> #include <string>
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS simdjson::dom::parser parser;
try { UNUSED simdjson::error_code error;
simdjson::dom::parser pj; UNUSED simdjson::dom::element elem;
auto result=pj.parse(Data, Size); parser.parse(Data, Size).tie(elem, error);
UNUSED auto v=result.value(); return 0;
}catch(...) {
}
#endif
return 0;
} }

19
fuzz/fuzz_print_json.cpp
View File

@ -7,14 +7,13 @@
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS simdjson::dom::parser parser;
try { simdjson::error_code error;
simdjson::dom::parser pj; simdjson::dom::element elem;
auto elem=pj.parse(Data, Size); parser.parse(Data, Size).tie(elem, error);
NulOStream os; if (!error) {
os<<elem; NulOStream os;
} catch (...) { os<<elem;
} }
#endif return 0;
return 0;
} }