From 0714f5fc672412d890d23f5679705c32e4928b6a Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 6 Apr 2020 16:04:37 -0700 Subject: [PATCH 1/7] Create dependencies/CMakeLists.txt --- CMakeLists.txt | 14 +------------- dependencies/CMakeLists.txt | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 dependencies/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index b294297e..c7d95310 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,23 +65,11 @@ add_definitions(-DSIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsone enable_testing() +add_subdirectory(dependencies) add_subdirectory(tools) add_subdirectory(tests) add_subdirectory(benchmark) -if (SIMDJSON_GOOGLE_BENCHMARKS) - if(NOT EXISTS dependencies/benchmark/CMakeLists.txt) - # message(STATUS "Unable to find dependencies/benchmark/CMakeLists.txt") - execute_process(COMMAND git submodule update --init -- dependencies/benchmark - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - option(BENCHMARK_ENABLE_TESTING OFF) - set(BENCHMARK_ENABLE_TESTING OFF) - option(BENCHMARK_ENABLE_INSTALL OFF) - set(BENCHMARK_ENABLE_INSTALL OFF) - add_subdirectory(dependencies/benchmark) -endif() - # for fuzzing, read the comments in the fuzz/CMakeLists.txt file option(ENABLE_FUZZING "enable building the fuzzers" ON) if(ENABLE_FUZZING) diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt new file mode 100644 index 00000000..2bef03f4 --- /dev/null +++ b/dependencies/CMakeLists.txt @@ -0,0 +1,22 @@ +# Init submodules so they can be used +find_package(Git QUIET) +if(GIT_FOUND) + option(SIMDJSON_GIT_SUBMODULE "Check submodules during build" ON) + if(SIMDJSON_GIT_SUBMODULE) + message(STATUS "Updating submodules") + execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE GIT_EXIT_CODE) + if(NOT GIT_EXIT_CODE EQUAL "0") + message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") + endif() + endif() +endif() + +if (SIMDJSON_GOOGLE_BENCHMARKS) + option(BENCHMARK_ENABLE_TESTING OFF) + set(BENCHMARK_ENABLE_TESTING OFF) + option(BENCHMARK_ENABLE_INSTALL OFF) + set(BENCHMARK_ENABLE_INSTALL OFF) + add_subdirectory(benchmark) +endif() From 6dabfa176ab402ba287a063db6c1c164615adad6 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 6 Apr 2020 17:18:37 -0700 Subject: [PATCH 2/7] Add competition libraries --- .gitignore | 1 + CMakeLists.txt | 7 +++-- benchmark/CMakeLists.txt | 1 + dependencies/CMakeLists.txt | 36 ++++++++++++++++++++++ tests/CMakeLists.txt | 61 +++++++++++++++++++++---------------- 5 files changed, 76 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 5dfa3ec6..d0c90473 100644 --- a/.gitignore +++ b/.gitignore @@ -119,6 +119,7 @@ objs /simdjson.h /singleheader/amalgamation_demo /singleheader/demo +/tests/allparserscheckfile /tests/basictests /tests/errortests /tests/integer_tests diff --git a/CMakeLists.txt b/CMakeLists.txt index c7d95310..59be9917 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ endif() project(simdjson DESCRIPTION "Parsing gigabytes of JSON per second" - LANGUAGES CXX + LANGUAGES CXX C ) # LTO seems to create all sorts of fun problems. Let us @@ -60,8 +60,9 @@ add_subdirectory(src) # # Compile tools / tests / benchmarks # -add_definitions(-DSIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/") -add_definitions(-DSIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/") +add_library(test-data INTERFACE) +target_compile_definitions(test-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/") +target_compile_definitions(test-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/") enable_testing() diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index a9be1807..89359b18 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -11,4 +11,5 @@ if (SIMDJSON_GOOGLE_BENCHMARKS) link_libraries(benchmark::benchmark) add_executable(bench_parse_call bench_parse_call.cpp) add_executable(bench_dom_api bench_dom_api.cpp) + target_link_libraries(bench_dom_api test-data) endif() \ No newline at end of file diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index 2bef03f4..8f5f9486 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -20,3 +20,39 @@ if (SIMDJSON_GOOGLE_BENCHMARKS) set(BENCHMARK_ENABLE_INSTALL OFF) add_subdirectory(benchmark) endif() + +add_library(competition-cJSON INTERFACE) +target_include_directories(competition-cJSON INTERFACE cJSON) + +add_library(competition-fastjson INTERFACE) +target_include_directories(competition-fastjson INTERFACE fastjson/src fastjson/include) + +add_library(competition-gason INTERFACE) +target_include_directories(competition-gason INTERFACE gason/src) + +add_library(competition-jsmn INTERFACE) +target_include_directories(competition-jsmn INTERFACE jsmn) + +add_library(competition-json INTERFACE) +target_include_directories(competition-json INTERFACE json/single_include) + +add_library(competition-json11 INTERFACE) +target_include_directories(competition-json11 INTERFACE json11) + +add_library(competition-jsoncppdist INTERFACE) +target_include_directories(competition-jsoncppdist INTERFACE jsoncppdist) + +add_library(competition-rapidjson INTERFACE) +target_include_directories(competition-rapidjson INTERFACE rapidjson/include) + +add_library(competition-sajson INTERFACE) +target_include_directories(competition-sajson INTERFACE sajson/include) + +add_library(competition-ujson4c ujson4c/src/ujdecode.c) +target_include_directories(competition-ujson4c PUBLIC ujson4c/3rdparty ujson4c/src) + +add_library(competition-core INTERFACE) +target_link_libraries(competition-core INTERFACE competition-json competition-rapidjson competition-sajson competition-cJSON competition-jsmn) + +add_library(competition-all INTERFACE) +target_link_libraries(competition-all INTERFACE competition-core competition-jsoncppdist competition-json11 competition-fastjson competition-gason competition-ujson4c) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2587d9af..9d589dcf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,9 +2,42 @@ function(add_cpp_test TEST_NAME TEST_FILE) add_executable(${TEST_NAME} ${TEST_FILE}) add_test(${TEST_NAME} ${TEST_NAME}) + target_link_libraries(${TEST_NAME} test-data) endfunction(add_cpp_test) -# Sets a target to only build when you run the test, and expect failure +# +# These test explicitly do #include "simdjson.cpp" +# +if (NOT MSVC) # Can't get simdjson-source to compile on Windows for some reason. + add_cpp_test(numberparsingcheck numberparsingcheck.cpp) + target_link_libraries(numberparsingcheck simdjson-source test-data simdjson-windows-headers) + add_cpp_test(stringparsingcheck stringparsingcheck.cpp) + target_link_libraries(stringparsingcheck simdjson-source test-data simdjson-windows-headers) +endif() + +# +# All remaining tests link with simdjson proper +# +link_libraries(simdjson) + +add_cpp_test(basictests basictests.cpp) +add_cpp_test(errortests errortests.cpp) +add_cpp_test(integer_tests integer_tests.cpp) +add_cpp_test(jsoncheck jsoncheck.cpp) +target_link_libraries(jsoncheck simdjson-windows-headers) +add_cpp_test(parse_many_test parse_many_test.cpp) +target_link_libraries(parse_many_test simdjson-windows-headers) +add_cpp_test(pointercheck pointercheck.cpp) + +# +# Competition parse test +# +add_executable(allparserscheckfile allparserscheckfile.cpp) +target_link_libraries(allparserscheckfile competition-all) + +# +# Compile-only tests +# function(add_compile_test TEST_NAME TEST_FILE EXPECT_SUCCESS) add_executable(${TEST_NAME} ${TEST_FILE}) set_target_properties(${TEST_NAME} PROPERTIES @@ -18,32 +51,6 @@ function(add_compile_test TEST_NAME TEST_FILE EXPECT_SUCCESS) endif() endfunction(add_compile_test) -# -# These test explicitly do #include "simdjson.cpp" -# -if (NOT MSVC) # Can't get simdjson-source to compile on Windows for some reason. - add_cpp_test(numberparsingcheck numberparsingcheck.cpp) - target_link_libraries(numberparsingcheck simdjson-source simdjson-windows-headers) - add_cpp_test(stringparsingcheck stringparsingcheck.cpp) - target_link_libraries(stringparsingcheck simdjson-source simdjson-windows-headers) -endif() - -# -# All remaining tests link with simdjson proper -# -link_libraries(simdjson) - -# add_executable(allparserscheckfile allparserscheckfile.cpp) -add_cpp_test(basictests basictests.cpp) -add_cpp_test(errortests errortests.cpp) -add_cpp_test(integer_tests integer_tests.cpp) -add_cpp_test(jsoncheck jsoncheck.cpp) -target_link_libraries(jsoncheck simdjson-windows-headers) -add_cpp_test(parse_many_test parse_many_test.cpp) -target_link_libraries(parse_many_test simdjson-windows-headers) -add_cpp_test(pointercheck pointercheck.cpp) - -# Compile-only tests # Don't add the tests if we're on VS2017 or older; they don't succeed. if(NOT (MSVC AND MSVC_VERSION LESS 1920)) add_compile_test(readme_examples readme_examples.cpp TRUE) From c1ff74c9a619bd988dbffbfce3c9cdb09c9a04cc Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 6 Apr 2020 17:35:11 -0700 Subject: [PATCH 3/7] Add issue150 test --- CMakeLists.txt | 1 + scripts/CMakeLists.txt | 5 +++++ scripts/issue150.sh | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 scripts/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 59be9917..7d6edd94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,7 @@ enable_testing() add_subdirectory(dependencies) add_subdirectory(tools) add_subdirectory(tests) +add_subdirectory(scripts) add_subdirectory(benchmark) # for fuzzing, read the comments in the fuzz/CMakeLists.txt file diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt new file mode 100644 index 00000000..aa55d1b1 --- /dev/null +++ b/scripts/CMakeLists.txt @@ -0,0 +1,5 @@ +add_test( + NAME issue150 + COMMAND ${PROJECT_SOURCE_DIR}/scripts/issue150.sh + WORKING_DIRECTORY $ +) \ No newline at end of file diff --git a/scripts/issue150.sh b/scripts/issue150.sh index be58fa64..3f390423 100755 --- a/scripts/issue150.sh +++ b/scripts/issue150.sh @@ -1,7 +1,6 @@ #!/bin/bash SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -cd $SCRIPTPATH/.. -for i in jsonchecker/adversarial/issue150/*.json ; do +for i in $SCRIPTPATH/../jsonchecker/adversarial/issue150/*.json ; do echo $i; ./allparserscheckfile -m $i; if [ $? -ne 0 ]; From 7b58fea91118e4cc1788dbaa56da1201a06b2136 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 6 Apr 2020 17:57:24 -0700 Subject: [PATCH 4/7] Add benchmark competitions to cmake --- .gitignore | 8 ++++++++ benchmark/CMakeLists.txt | 28 ++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index d0c90473..6fd32b0f 100644 --- a/.gitignore +++ b/.gitignore @@ -57,11 +57,19 @@ objs /allparsingcompetition /basictests /benchfeatures +/benchmark/allparsingcompetition /benchmark/bench_dom_api /benchmark/bench_parse_call +/benchmark/distinctuseridcompetition /benchmark/get_corpus_benchmark +/benchmark/minifiercompetition /benchmark/parse +/benchmark/parseandstatcompetition +/benchmark/parse_nonumberparsing +/benchmark/parse_nostringparsing +/benchmark/parse_noutf8validation /benchmark/parse_stream +/benchmark/parsingcompetition /benchmark/perfdiff /benchmark/statisticalmodel /build/ diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 89359b18..5864a99e 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,10 +1,18 @@ include_directories( . linux ) link_libraries(simdjson) -add_executable(parse parse.cpp) -add_executable(statisticalmodel statisticalmodel.cpp) -add_executable(parse_stream parse_stream.cpp) +# add_executable(benchfeatures benchfeatures.cpp) add_executable(get_corpus_benchmark get_corpus_benchmark.cpp) add_executable(perfdiff perfdiff.cpp) +add_executable(parse parse.cpp) +add_executable(parse_stream parse_stream.cpp) +add_executable(statisticalmodel statisticalmodel.cpp) + +add_executable(parse_noutf8validation parse.cpp) +target_compile_definitions(parse_noutf8validation PRIVATE SIMDJSON_SKIPUTF8VALIDATION) +add_executable(parse_nonumberparsing parse.cpp) +target_compile_definitions(parse_nonumberparsing PRIVATE SIMDJSON_SKIPNUMBERPARSING) +add_executable(parse_nostringparsing parse.cpp) +target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARSING) # Google Benchmarks if (SIMDJSON_GOOGLE_BENCHMARKS) @@ -12,4 +20,16 @@ if (SIMDJSON_GOOGLE_BENCHMARKS) add_executable(bench_parse_call bench_parse_call.cpp) add_executable(bench_dom_api bench_dom_api.cpp) target_link_libraries(bench_dom_api test-data) -endif() \ No newline at end of file +endif() + +add_executable(distinctuseridcompetition distinctuseridcompetition.cpp) +target_link_libraries(distinctuseridcompetition competition-core) +add_executable(minifiercompetition minifiercompetition.cpp) +target_link_libraries(minifiercompetition competition-core) +add_executable(parseandstatcompetition parseandstatcompetition.cpp) +target_link_libraries(parseandstatcompetition competition-core) +add_executable(parsingcompetition parsingcompetition.cpp) +target_link_libraries(parsingcompetition competition-core) +add_executable(allparsingcompetition parsingcompetition.cpp) +target_link_libraries(allparsingcompetition competition-all) +target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER) From 7317fe1440ba8d80ef1e1cc5719e9378d97fd47e Mon Sep 17 00:00:00 2001 From: John Keiser Date: Tue, 7 Apr 2020 09:25:11 -0700 Subject: [PATCH 5/7] Don't reinitialize submodules Add ability to turn competitive benchmarks off (no need for submodules) --- .drone.yml | 59 ++++++++++++++-------------- CMakeLists.txt | 1 + benchmark/CMakeLists.txt | 25 ++++++------ dependencies/CMakeLists.txt | 78 +++++++++++++++++++++---------------- scripts/CMakeLists.txt | 12 +++--- tests/CMakeLists.txt | 6 ++- 6 files changed, 100 insertions(+), 81 deletions(-) diff --git a/.drone.yml b/.drone.yml index 51d26504..81440d8d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -226,35 +226,36 @@ steps: CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF commands: - apt-get update -qq - - apt-get install -y clang make cmake + - apt-get install -y clang make cmake git + - $CC --version + - mkdir build && cd build + - cmake $CMAKE_FLAGS .. + - make -j + - ctest --output-on-failure +--- +kind: pipeline +name: amd64_clang_cmake_no_exceptions + +platform: + os: linux + arch: amd64 + +steps: +- name: Build and Test + image: ubuntu:18.04 + environment: + CC: clang + CXX: clang++ + CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + commands: + - apt-get update -qq + - apt-get install -y clang make cmake git - $CC --version - mkdir build && cd build - cmake $CMAKE_FLAGS .. - make -j - ctest --output-on-failure --- - kind: pipeline - name: amd64_clang_cmake_no_exceptions - - platform: - os: linux - arch: amd64 - - steps: - - name: Build and Test - image: ubuntu:18.04 - environment: - CC: clang - CXX: clang++ - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF - commands: - - apt-get update -qq - - apt-get install -y clang make cmake - - $CC --version - - mkdir build && cd build - - cmake $CMAKE_FLAGS .. - - make -j - - ctest --output-on-failure kind: pipeline name: amd64_clang_cmake_static @@ -271,7 +272,7 @@ steps: CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON commands: - apt-get update -qq - - apt-get install -y clang make cmake + - apt-get install -y clang make cmake git - $CC --version - mkdir build && cd build - cmake $CMAKE_FLAGS .. @@ -340,7 +341,7 @@ steps: CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF commands: - apt-get update -qq - - apt-get install -y clang make cmake + - apt-get install -y clang make cmake git - $CC --version - mkdir build && cd build - cmake $CMAKE_FLAGS .. @@ -386,7 +387,7 @@ steps: CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON commands: - apt-get update -qq - - apt-get install -y clang make cmake + - apt-get install -y clang make cmake git - $CC --version - mkdir build && cd build - cmake $CMAKE_FLAGS .. @@ -433,7 +434,7 @@ steps: CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON commands: - apt-get update -qq - - apt-get install -y clang make cmake + - apt-get install -y clang make cmake git - $CC --version - mkdir build && cd build - cmake $CMAKE_FLAGS .. @@ -480,7 +481,7 @@ steps: CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON commands: - apt-get update -y - - apt-get install -y make $CC g++ cmake + - apt-get install -y make $CC g++ cmake git - $CC --version - mkdir build && cd build - cmake $CMAKE_FLAGS .. @@ -527,7 +528,7 @@ steps: CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION_HASWELL=OFF commands: - apt-get update -qq - - apt-get install -y clang make cmake + - apt-get install -y clang make cmake git - $CC --version - mkdir build && cd build - cmake $CMAKE_FLAGS .. diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d6edd94..f5898e81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ set(PROJECT_VERSION_PATCH 1) set(SIMDJSON_LIB_VERSION "0.3.1" CACHE STRING "simdjson library version") set(SIMDJSON_LIB_SOVERSION "1" CACHE STRING "simdjson library soversion") +option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON) option(SIMDJSON_IMPLEMENTATION_HASWELL "Include the haswell implementation" ON) option(SIMDJSON_IMPLEMENTATION_WESTMERE "Include the westmere implementation" ON) option(SIMDJSON_IMPLEMENTATION_ARM64 "Include the arm64 implementation" ON) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 5864a99e..523f0773 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -14,7 +14,6 @@ target_compile_definitions(parse_nonumberparsing PRIVATE SIMDJSON_SKIPNUMBERPARS add_executable(parse_nostringparsing parse.cpp) target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARSING) -# Google Benchmarks if (SIMDJSON_GOOGLE_BENCHMARKS) link_libraries(benchmark::benchmark) add_executable(bench_parse_call bench_parse_call.cpp) @@ -22,14 +21,16 @@ if (SIMDJSON_GOOGLE_BENCHMARKS) target_link_libraries(bench_dom_api test-data) endif() -add_executable(distinctuseridcompetition distinctuseridcompetition.cpp) -target_link_libraries(distinctuseridcompetition competition-core) -add_executable(minifiercompetition minifiercompetition.cpp) -target_link_libraries(minifiercompetition competition-core) -add_executable(parseandstatcompetition parseandstatcompetition.cpp) -target_link_libraries(parseandstatcompetition competition-core) -add_executable(parsingcompetition parsingcompetition.cpp) -target_link_libraries(parsingcompetition competition-core) -add_executable(allparsingcompetition parsingcompetition.cpp) -target_link_libraries(allparsingcompetition competition-all) -target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER) +if (SIMDJSON_COMPETITION) + add_executable(distinctuseridcompetition distinctuseridcompetition.cpp) + target_link_libraries(distinctuseridcompetition competition-core) + add_executable(minifiercompetition minifiercompetition.cpp) + target_link_libraries(minifiercompetition competition-core) + add_executable(parseandstatcompetition parseandstatcompetition.cpp) + target_link_libraries(parseandstatcompetition competition-core) + add_executable(parsingcompetition parsingcompetition.cpp) + target_link_libraries(parsingcompetition competition-core) + add_executable(allparsingcompetition parsingcompetition.cpp) + target_link_libraries(allparsingcompetition competition-all) + target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER) +endif() diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index 8f5f9486..85411db9 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -1,58 +1,70 @@ -# Init submodules so they can be used -find_package(Git QUIET) -if(GIT_FOUND) - option(SIMDJSON_GIT_SUBMODULE "Check submodules during build" ON) - if(SIMDJSON_GIT_SUBMODULE) - message(STATUS "Updating submodules") - execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive +# Initializes a git submodule if it hasn't been initialized before +# Does NOT attempt to update or otherwise modify git submodules that are already initialized. +function(initialize_submodule DIRECTORY) + if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/.git) + find_package(Git QUIET REQUIRED) + message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/.git does not exist. Initializing ${DIRECTORY} submodule ...") + execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init ${DIRECTORY} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE GIT_EXIT_CODE) if(NOT GIT_EXIT_CODE EQUAL "0") - message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") + message(FATAL_ERROR "${GIT_EXECUTABLE} submodule update --init dependencies/${DIRECTORY} failed with exit code ${GIT_EXIT_CODE}, please checkout submodules") endif() endif() -endif() +endfunction(initialize_submodule) if (SIMDJSON_GOOGLE_BENCHMARKS) option(BENCHMARK_ENABLE_TESTING OFF) set(BENCHMARK_ENABLE_TESTING OFF) option(BENCHMARK_ENABLE_INSTALL OFF) set(BENCHMARK_ENABLE_INSTALL OFF) + initialize_submodule(benchmark) add_subdirectory(benchmark) endif() -add_library(competition-cJSON INTERFACE) -target_include_directories(competition-cJSON INTERFACE cJSON) +if (SIMDJSON_COMPETITION) + initialize_submodule(cJSON) + add_library(competition-cJSON INTERFACE) + target_include_directories(competition-cJSON INTERFACE cJSON) -add_library(competition-fastjson INTERFACE) -target_include_directories(competition-fastjson INTERFACE fastjson/src fastjson/include) + initialize_submodule(fastjson) + add_library(competition-fastjson INTERFACE) + target_include_directories(competition-fastjson INTERFACE fastjson/src fastjson/include) -add_library(competition-gason INTERFACE) -target_include_directories(competition-gason INTERFACE gason/src) + initialize_submodule(gason) + add_library(competition-gason INTERFACE) + target_include_directories(competition-gason INTERFACE gason/src) -add_library(competition-jsmn INTERFACE) -target_include_directories(competition-jsmn INTERFACE jsmn) + initialize_submodule(jsmn) + add_library(competition-jsmn INTERFACE) + target_include_directories(competition-jsmn INTERFACE jsmn) -add_library(competition-json INTERFACE) -target_include_directories(competition-json INTERFACE json/single_include) + initialize_submodule(json) + add_library(competition-json INTERFACE) + target_include_directories(competition-json INTERFACE json/single_include) -add_library(competition-json11 INTERFACE) -target_include_directories(competition-json11 INTERFACE json11) + initialize_submodule(json11) + add_library(competition-json11 INTERFACE) + target_include_directories(competition-json11 INTERFACE json11) -add_library(competition-jsoncppdist INTERFACE) -target_include_directories(competition-jsoncppdist INTERFACE jsoncppdist) + add_library(competition-jsoncppdist INTERFACE) + target_include_directories(competition-jsoncppdist INTERFACE jsoncppdist) -add_library(competition-rapidjson INTERFACE) -target_include_directories(competition-rapidjson INTERFACE rapidjson/include) + initialize_submodule(rapidjson) + add_library(competition-rapidjson INTERFACE) + target_include_directories(competition-rapidjson INTERFACE rapidjson/include) -add_library(competition-sajson INTERFACE) -target_include_directories(competition-sajson INTERFACE sajson/include) + initialize_submodule(sajson) + add_library(competition-sajson INTERFACE) + target_include_directories(competition-sajson INTERFACE sajson/include) -add_library(competition-ujson4c ujson4c/src/ujdecode.c) -target_include_directories(competition-ujson4c PUBLIC ujson4c/3rdparty ujson4c/src) + initialize_submodule(ujson4c) + add_library(competition-ujson4c ujson4c/src/ujdecode.c) + target_include_directories(competition-ujson4c PUBLIC ujson4c/3rdparty ujson4c/src) -add_library(competition-core INTERFACE) -target_link_libraries(competition-core INTERFACE competition-json competition-rapidjson competition-sajson competition-cJSON competition-jsmn) + add_library(competition-core INTERFACE) + target_link_libraries(competition-core INTERFACE competition-json competition-rapidjson competition-sajson competition-cJSON competition-jsmn) -add_library(competition-all INTERFACE) -target_link_libraries(competition-all INTERFACE competition-core competition-jsoncppdist competition-json11 competition-fastjson competition-gason competition-ujson4c) + add_library(competition-all INTERFACE) + target_link_libraries(competition-all INTERFACE competition-core competition-jsoncppdist competition-json11 competition-fastjson competition-gason competition-ujson4c) +endif() \ No newline at end of file diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index aa55d1b1..2e5d6527 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -1,5 +1,7 @@ -add_test( - NAME issue150 - COMMAND ${PROJECT_SOURCE_DIR}/scripts/issue150.sh - WORKING_DIRECTORY $ -) \ No newline at end of file +if (SIMDJSON_COMPETITION) + add_test( + NAME issue150 + COMMAND ${PROJECT_SOURCE_DIR}/scripts/issue150.sh + WORKING_DIRECTORY $ + ) +endif() \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9d589dcf..195b011d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -32,8 +32,10 @@ add_cpp_test(pointercheck pointercheck.cpp) # # Competition parse test # -add_executable(allparserscheckfile allparserscheckfile.cpp) -target_link_libraries(allparserscheckfile competition-all) +if (SIMDJSON_COMPETITION) + add_executable(allparserscheckfile allparserscheckfile.cpp) + target_link_libraries(allparserscheckfile competition-all) +endif() # # Compile-only tests From 6835dd73bc0b31274b6e4eb7d724f3411d7e521a Mon Sep 17 00:00:00 2001 From: John Keiser Date: Tue, 7 Apr 2020 12:40:25 -0700 Subject: [PATCH 6/7] Only apply compile flags to simdjson --- CMakeLists.txt | 19 +++++------- README.md | 3 +- include/CMakeLists.txt | 69 ++++++++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 60 ++++++++++++------------------------ 4 files changed, 99 insertions(+), 52 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f5898e81..1b718701 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,5 @@ cmake_minimum_required(VERSION 3.9) # CMP0069 NEW +include(GNUInstallDirs) message(STATUS "cmake version ${CMAKE_VERSION}") if (NOT CMAKE_BUILD_TYPE) @@ -32,24 +33,15 @@ set(SIMDJSON_LIB_VERSION "0.3.1" CACHE STRING "simdjson library version") set(SIMDJSON_LIB_SOVERSION "1" CACHE STRING "simdjson library soversion") option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON) -option(SIMDJSON_IMPLEMENTATION_HASWELL "Include the haswell implementation" ON) -option(SIMDJSON_IMPLEMENTATION_WESTMERE "Include the westmere implementation" ON) -option(SIMDJSON_IMPLEMENTATION_ARM64 "Include the arm64 implementation" ON) -option(SIMDJSON_IMPLEMENTATION_FALLBACK "Include the fallback implementation" ON) if(NOT MSVC) option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library else() option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library endif() -option(SIMDJSON_SANITIZE "Sanitize addresses" OFF) option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" OFF) -option(SIMDJSON_ENABLE_THREADS "enable threaded operation" ON) -option(SIMDJSON_NO_EXCEPTIONS "Disable simdjson's exception-throwing interface" OFF) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake") -find_package(Options) - # # Create the top level simdjson library (must be done at this level to use both src/ and include/ # directories) @@ -61,12 +53,12 @@ add_subdirectory(src) # # Compile tools / tests / benchmarks # +enable_testing() + add_library(test-data INTERFACE) target_compile_definitions(test-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/") target_compile_definitions(test-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/") -enable_testing() - add_subdirectory(dependencies) add_subdirectory(tools) add_subdirectory(tests) @@ -79,6 +71,11 @@ if(ENABLE_FUZZING) add_subdirectory(fuzz) endif() +if(${CMAKE_C_COMPILER_ID} MATCHES "Intel") # icc / icpc + # prevent shared libraries from depending on Intel provided libraries + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel") +endif() + set(CPACK_PACKAGE_VENDOR "Daniel Lemire") set(CPACK_PACKAGE_CONTACT "lemire@gmail.com") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Parsing gigabytes of JSON per second") diff --git a/README.md b/README.md index 98523548..e2c3353a 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,8 @@ For the video inclined,
Funding ------- -The work is supported by the Natural Sciences and Engineering Research Council of Canada under grant number RGPIN-2017-03910. +The work is supported by the Natural Sciences and Engineering Research Council of Canada under grant +number RGPIN-2017-03910. [license]: LICENSE [license img]: https://img.shields.io/badge/License-Apache%202-blue.svg diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 7e06d982..23d1373e 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -1,2 +1,71 @@ +# +# simdjson headers and flags required to compile them +# + +add_library(simdjson-headers INTERFACE) + +# Include directory +target_include_directories(simdjson-headers INTERFACE + $ + $) + +# Flags absolutely needed to compile simdjson at all +target_compile_features(simdjson-headers INTERFACE cxx_std_17) +if(MSVC) # Windows + # C++ standard flags + target_compile_options(simdjson-headers INTERFACE /std:c++17) + # Base flags + target_compile_options(simdjson-headers INTERFACE /nologo) + # Warning flags + target_compile_options(simdjson-headers INTERFACE /W3 /D_CRT_SECURE_NO_WARNINGS /wd4005 /wd4996 /wd4267 /wd4244 /wd4113) +else() # Linux + # Base flags + target_compile_options(simdjson-headers INTERFACE -fPIC) + # C++ standard flags + target_compile_options(simdjson-headers INTERFACE -std=c++17) + # Warning flags + target_compile_options(simdjson-headers INTERFACE -Wall -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self) + # Debug and release specific flags + target_compile_options(simdjson-headers INTERFACE $<$:-ggdb>) + target_compile_options(simdjson-headers INTERFACE $<$:-O3 -DNDEBUG>) +endif() + + +# Optional flags +option(SIMDJSON_IMPLEMENTATION_HASWELL "Include the haswell implementation" ON) +option(SIMDJSON_IMPLEMENTATION_WESTMERE "Include the westmere implementation" ON) +option(SIMDJSON_IMPLEMENTATION_ARM64 "Include the arm64 implementation" ON) +option(SIMDJSON_IMPLEMENTATION_FALLBACK "Include the fallback implementation" ON) + +option(SIMDJSON_EXCEPTIONS "Enable simdjson's exception-throwing interface" ON) +if(NOT SIMDJSON_EXCEPTIONS) + message(STATUS "simdjson exception interface turned off. Code that does not check error codes will not compile.") + target_compile_definitions(simdjson-headers INTERFACE SIMDJSON_EXCEPTIONS=0) +endif() + +option(SIMDJSON_ENABLE_THREADS "Enable threaded operation" ON) +if(SIMDJSON_ENABLE_THREADS) + find_package(Threads REQUIRED) + target_link_libraries(simdjson-headers INTERFACE Threads::Threads) +endif() + +option(SIMDJSON_SANITIZE "Sanitize addresses" OFF) +if(SIMDJSON_SANITIZE) + # Not sure which + target_compile_options(simdjson-headers INTERFACE -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all) + target_link_libraries(simdjson-headers INTERFACE -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all) + + # Ubuntu bug for GCC 5.0+ (safe for all versions) + if (CMAKE_COMPILER_IS_GNUCC) + target_link_libraries(simdjson-headers INTERFACE -fuse-ld=gold) + endif() +endif() + +install(TARGETS simdjson-headers + EXPORT simdjson-headers-config + INCLUDES DESTINATION include) +install(EXPORT simdjson-headers-config + FILE simdjson-headers-config.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/simdjson) install(DIRECTORY simdjson DESTINATION include FILES_MATCHING PATTERN *.h) install(FILES simdjson.h DESTINATION include) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 70270fbe..6bcbccdf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,6 @@ -include(GNUInstallDirs) +# +# simdjson as a library +# if(SIMDJSON_BUILD_STATIC) MESSAGE( STATUS "Building a static library." ) @@ -6,34 +8,34 @@ if(SIMDJSON_BUILD_STATIC) else() MESSAGE( STATUS "Building a dynamic library." ) add_library(simdjson SHARED "") + target_compile_definitions(simdjson INTERFACE SIMDJSON_USING_LIBRARY=1) + if(MSVC) + MESSAGE( STATUS "Building a Windows DLL using Visual Studio, exporting all symbols automatically." ) + set_target_properties(simdjson PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1) + endif() endif() target_sources(simdjson PRIVATE simdjson.cpp) +target_link_libraries(simdjson PUBLIC simdjson-headers) target_include_directories(simdjson PRIVATE .) -target_include_directories(simdjson PUBLIC $) -if(SIMDJSON_NO_EXCEPTIONS) - message(STATUS "simdjson exception interface turned off. Code that does not check error codes will not compile.") - target_compile_definitions(simdjson PUBLIC SIMDJSON_EXCEPTIONS=0) +if(NOT MSVC) + ## We output the library at the root of the current directory where cmake is invoked + ## This is handy but Visual Studio will happily ignore us + set_target_properties(simdjson PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) + MESSAGE( STATUS "Library output directory (does not apply to Visual Studio): " ${CMAKE_BINARY_DIR}) endif() -if(${CMAKE_C_COMPILER_ID} MATCHES "Intel") # icc / icpc - # prevent shared libraries from depending on Intel provided libraries - target_link_options(simdjson PUBLIC "-static-intel") -endif() - -if(SIMDJSON_ENABLE_THREADS) - find_package(Threads REQUIRED) - target_link_libraries(simdjson PRIVATE Threads::Threads) -endif() - -# For targets that want to include the source directly +# +# simdjson to be compiled into your exe as source (you must #include simdjson.cpp) +# add_library(simdjson-source INTERFACE) +target_link_libraries(simdjson-source INTERFACE simdjson-headers) target_include_directories(simdjson-source INTERFACE .) -target_include_directories(simdjson-source INTERFACE $) +# # Installation - +# install(TARGETS simdjson EXPORT simdjson-config ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -46,25 +48,3 @@ install(EXPORT simdjson-config NAMESPACE simdjson:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/simdjson ) - -if(NOT MSVC) -## We output the library at the root of the current directory where cmake is invoked -## This is handy but Visual Studio will happily ignore us -set_target_properties(simdjson PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) -MESSAGE( STATUS "Library output directory (does not apply to Visual Studio): " ${CMAKE_BINARY_DIR}) -endif() - -if(NOT SIMDJSON_BUILD_STATIC) - target_compile_definitions(simdjson INTERFACE SIMDJSON_USING_LIBRARY=1) -endif() - -if(MSVC AND (NOT SIMDJSON_BUILD_STATIC)) - if (CMAKE_VERSION VERSION_LESS 3.4) - MESSAGE( STATUS "To build a Windows DLL using Visual Studio, you may need cmake 3.4 or better." ) - endif() - MESSAGE( STATUS "Building a Windows DLL using Visual Studio, exporting all symbols automatically." ) - set_target_properties(simdjson - PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS 1) -endif() - From 3c91690e552c1c6175cf9343c5d9ea1a39e61d57 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Wed, 8 Apr 2020 18:13:01 -0700 Subject: [PATCH 7/7] Don't run parsing competitions on Windows (They currently use linux-specific stuff.) --- CMakeLists.txt | 9 +++---- tools/cmake/FindOptions.cmake | 44 ----------------------------------- 2 files changed, 5 insertions(+), 48 deletions(-) delete mode 100644 tools/cmake/FindOptions.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b718701..83bcc36f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,11 +32,12 @@ set(PROJECT_VERSION_PATCH 1) set(SIMDJSON_LIB_VERSION "0.3.1" CACHE STRING "simdjson library version") set(SIMDJSON_LIB_SOVERSION "1" CACHE STRING "simdjson library soversion") -option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON) -if(NOT MSVC) - option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library -else() +if(MSVC) option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library + option(SIMDJSON_COMPETITION "Compile competitive benchmarks" OFF) + else() + option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library + option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON) endif() option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" OFF) diff --git a/tools/cmake/FindOptions.cmake b/tools/cmake/FindOptions.cmake deleted file mode 100644 index 50e89692..00000000 --- a/tools/cmake/FindOptions.cmake +++ /dev/null @@ -1,44 +0,0 @@ -macro(append var string) - set(${var} "${${var}} ${string}") -endmacro(append) - -set(SANITIZE_FLAGS "") -if(SIMDJSON_SANITIZE) - set(SIMDJSON_SANITIZE_FLAGS "-fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=all") - if (CMAKE_COMPILER_IS_GNUCC) - # Ubuntu bug for GCC 5.0+ (safe for all versions) - append(CMAKE_EXE_LINKER_FLAGS "-fuse-ld=gold") - append(CMAKE_SHARED_LINKER_FLAGS "-fuse-ld=gold") - endif() -endif() - -if (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)|(x86_64)") - set (X64 TRUE) -else () - set (X64 FALSE) -endif () - -if(MSVC) -set(CXXSTD_FLAGS "/std:c++17") -else() -set(CXXSTD_FLAGS "-std=c++17 -fPIC") -endif() - -set(WARNING_FLAGS "-Wall") -if(NOT MSVC) -set(WARNING_FLAGS "${WARNING_FLAGS} -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self") -set(CMAKE_C_FLAGS_DEBUG "-ggdb") -set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") -endif() - -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${STD_FLAGS} ${OPT_FLAGS} ${INCLUDE_FLAGS} ${WARNING_FLAGS} ${SIMDJSON_SANITIZE_FLAGS} ") - -if(NOT MSVC) -set(CMAKE_CXX_FLAGS_DEBUG "-ggdb") -set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") -endif() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXXSTD_FLAGS} ${OPT_FLAGS} ${INCLUDE_FLAGS} ${WARNING_FLAGS} ${SIMDJSON_SANITIZE_FLAGS} ") - -if(MSVC) -add_definitions( "${OPT_FLAGS} /W3 /D_CRT_SECURE_NO_WARNINGS /wd4005 /wd4996 /wd4267 /wd4244 /wd4113 /nologo") -endif()