jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Make fuzzing work without exceptions

This commit is contained in:
John Keiser 2020-04-16 18:49:10 -07:00
parent fbf274a42b
commit db314bc381
8 changed files with 125 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/config.yml
View File

@ -102,7 +102,7 @@ jobs:
sanitize-gcc9:
description: Build and run tests on GCC 9 and AVX 2 with a cmake sanitize build
executor: gcc9
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
steps: [ cmake_test_all ]
sanitize-clang6:
description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build

7
CMakeLists.txt
View File

@ -161,12 +161,7 @@ add_subdirectory(dependencies)
add_subdirectory(tests)
add_subdirectory(examples)
add_subdirectory(benchmark)
# for fuzzing, read the comments in the fuzz/CMakeLists.txt file
option(ENABLE_FUZZING "enable building the fuzzers" ON)
if(ENABLE_FUZZING AND SIMDJSON_EXCEPTIONS)
add_subdirectory(fuzz)
endif()
add_subdirectory(fuzz)
#
# CPack

93
fuzz/CMakeLists.txt
View File

@ -1,56 +1,59 @@
# First attempt at a fuzzer, using libFuzzer.
#
# compile like this:
# mkdir build-fuzzer
# cd build-fuzzer
# export LDFLAGS="-fsanitize=address,undefined"
# export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CXX=clang++
# export CC=clang++
# cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer
# ninja
option(ENABLE_FUZZING "enable building the fuzzers" ON)
if(ENABLE_FUZZING)
# settings this links in a main. useful for reproducing,
# kcov, gdb, afl, valgrind.
# (note that libFuzzer can also reproduce, just pass it the files)
#
# Using this by default, means the fuzzers will be built as a part of the normal
# workflow, meaning they wont bitrot and will participate in refactoring etc.
#
option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On)
# First attempt at a fuzzer, using libFuzzer.
#
# compile like this:
# mkdir build-fuzzer
# cd build-fuzzer
# export LDFLAGS="-fsanitize=address,undefined"
# export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CXX=clang++
# export CC=clang++
# cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer
# ninja
# For oss-fuzz - insert $LIB_FUZZING_ENGINE into the link flags, but only for
# the fuzz targets, otherwise the cmake configuration step fails.
set(SIMDJSON_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets")
# settings this links in a main. useful for reproducing,
# kcov, gdb, afl, valgrind.
# (note that libFuzzer can also reproduce, just pass it the files)
#
# Using this by default, means the fuzzers will be built as a part of the normal
# workflow, meaning they wont bitrot and will participate in refactoring etc.
#
option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On)
set(SOURCES
fuzz_parser.cpp
fuzz_minify.cpp
fuzz_dump.cpp
fuzz_print_json.cpp
fuzz_dump_raw_tape.cpp
)
# For oss-fuzz - insert $LIB_FUZZING_ENGINE into the link flags, but only for
# the fuzz targets, otherwise the cmake configuration step fails.
set(SIMDJSON_FUZZ_LDFLAGS "" CACHE STRING "LDFLAGS for the fuzz targets")
add_custom_target(print_all_fuzz_targets
add_custom_target(print_all_fuzz_targets
COMMAND ${CMAKE_COMMAND} -E echo ${SOURCES}
)
)
macro(implement_fuzzer sourcefile)
get_filename_component(basename ${sourcefile} NAME_WE)
set(name ${basename})
add_executable(${name} ${sourcefile})
# Fuzzer build flags and libraries
add_library(simdjson-fuzzer INTERFACE)
if (SIMDJSON_FUZZ_LINKMAIN)
target_sources(${name} PRIVATE main.cpp)
target_link_libraries(simdjson-fuzzer INTERFACE simdjson-source)
target_sources(simdjson-fuzzer INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/main.cpp)
else ()
target_link_libraries(simdjson-fuzzer INTERFACE simdjson)
endif ()
target_link_libraries(${name} PRIVATE simdjson simdjson-flags)
if (SIMDJSON_FUZZ_LDFLAGS)
target_link_libraries(${name} PRIVATE ${SIMDJSON_FUZZ_LDFLAGS})
endif ()
endmacro ()
target_link_libraries(simdjson-fuzzer INTERFACE simdjson-flags)
target_link_libraries(simdjson-fuzzer INTERFACE ${SIMDJSON_FUZZ_LDFLAGS})
function(implement_fuzzer name)
add_executable(${name} ${name}.cpp)
target_link_libraries(${name} PRIVATE simdjson-fuzzer)
add_test(${name} ${name})
set_property(TEST ${name} APPEND PROPERTY LABELS fuzz)
endfunction()
foreach (X IN ITEMS ${SOURCES})
implement_fuzzer(${X})
endforeach ()
implement_fuzzer(fuzz_parser)
implement_fuzzer(fuzz_minify)
implement_fuzzer(fuzz_dump)
implement_fuzzer(fuzz_print_json)
implement_fuzzer(fuzz_dump_raw_tape)
endif()

96
fuzz/fuzz_dump.cpp
View File

@ -9,57 +9,53 @@
// example from doc/basics.md#tree-walking-and-json-element-types
static void print_json(std::ostream& os, simdjson::dom::element element) {
#if SIMDJSON_EXCEPTIONS
const char endl='\n';
switch (element.type()) {
case simdjson::dom::element_type::ARRAY:
os << "[";
for (simdjson::dom::element child : simdjson::dom::array(element)) {
print_json(os, child);
os << ",";
}
os << "]";
break;
case simdjson::dom::element_type::OBJECT:
os << "{";
for (simdjson::dom::key_value_pair field : simdjson::dom::object(element)) {
os << "\"" << field.key << "\": ";
print_json(os, field.value);
}
os << "}";
break;
case simdjson::dom::element_type::INT64:
os << int64_t(element) << endl;
break;
case simdjson::dom::element_type::UINT64:
os << uint64_t(element) << endl;
break;
case simdjson::dom::element_type::DOUBLE:
os << double(element) << endl;
break;
case simdjson::dom::element_type::STRING:
os << std::string_view(element) << endl;
break;
case simdjson::dom::element_type::BOOL:
os << bool(element) << endl;
break;
case simdjson::dom::element_type::NULL_VALUE:
os << "null" << endl;
break;
const char endl='\n';
switch (element.type()) {
case simdjson::dom::element_type::ARRAY:
os << "[";
for (simdjson::dom::element child : element.get<simdjson::dom::array>().first) {
print_json(os, child);
os << ",";
}
#endif
os << "]";
break;
case simdjson::dom::element_type::OBJECT:
os << "{";
for (simdjson::dom::key_value_pair field : element.get<simdjson::dom::object>().first) {
os << "\"" << field.key << "\": ";
print_json(os, field.value);
}
os << "}";
break;
case simdjson::dom::element_type::INT64:
os << element.get<int64_t>().first << endl;
break;
case simdjson::dom::element_type::UINT64:
os << element.get<uint64_t>().first << endl;
break;
case simdjson::dom::element_type::DOUBLE:
os << element.get<double>().first << endl;
break;
case simdjson::dom::element_type::STRING:
os << element.get<std::string_view>().first << endl;
break;
case simdjson::dom::element_type::BOOL:
os << element.get<bool>().first << endl;
break;
case simdjson::dom::element_type::NULL_VALUE:
os << "null" << endl;
break;
}
}
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS
try {
simdjson::dom::parser pj;
auto elem=pj.parse(Data, Size);
auto v=elem.value();
NulOStream os;
//std::ostream& os(std::cout);
print_json(os,v);
} catch (...) {
}
#endif
return 0;
simdjson::dom::parser parser;
simdjson::error_code error;
simdjson::dom::element elem;
parser.parse(Data, Size).tie(elem, error);
if (error) { return 1; }
NulOStream os;
//std::ostream& os(std::cout);
print_json(os,elem);
return 0;
}

18
fuzz/fuzz_dump_raw_tape.cpp
View File

@ -7,15 +7,13 @@
#include "NullBuffer.h"
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS
try {
simdjson::dom::parser pj;
auto elem=pj.parse(Data, Size);
auto v=elem.value();
NulOStream os;
UNUSED auto dumpstatus=v.dump_raw_tape(os);
} catch (...) {
}
#endif
simdjson::dom::parser parser;
simdjson::error_code error;
simdjson::dom::element elem;
parser.parse(Data, Size).tie(elem, error);
if (error) { return 1; }
NulOStream os;
UNUSED auto dumpstatus = elem.dump_raw_tape(os);
return 0;
}

16
fuzz/fuzz_minify.cpp
View File

@ -8,15 +8,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
auto end = begin + Size;
std::string str(begin, end);
#if SIMDJSON_EXCEPTIONS
try {
simdjson::dom::parser parser;
simdjson::dom::element doc = parser.parse(str);
std::string minified=simdjson::minify(doc);
(void)minified;
} catch (...) {
simdjson::dom::parser parser;
simdjson::error_code error;
simdjson::dom::element elem;
parser.parse(str).tie(elem, error);
if (error) { return 1; }
}
#endif
std::string minified=simdjson::minify(elem);
(void)minified;
return 0;
}

14
fuzz/fuzz_parser.cpp
View File

@ -3,13 +3,9 @@
#include <cstdint>
#include <string>
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS
try {
simdjson::dom::parser pj;
auto result=pj.parse(Data, Size);
UNUSED auto v=result.value();
}catch(...) {
}
#endif
return 0;
simdjson::dom::parser parser;
UNUSED simdjson::error_code error;
UNUSED simdjson::dom::element elem;
parser.parse(Data, Size).tie(elem, error);
return 0;
}

19
fuzz/fuzz_print_json.cpp
View File

@ -7,14 +7,13 @@
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
#if SIMDJSON_EXCEPTIONS
try {
simdjson::dom::parser pj;
auto elem=pj.parse(Data, Size);
NulOStream os;
os<<elem;
} catch (...) {
}
#endif
return 0;
simdjson::dom::parser parser;
simdjson::error_code error;
simdjson::dom::element elem;
parser.parse(Data, Size).tie(elem, error);
if (!error) {
NulOStream os;
os<<elem;
}
return 0;
}