Remove git modules (#1258)

* Bump minimum CMake version

* Remove unnecessary git checks

* Move benchmark options where they are used

* Declare helper functions for dependencies

The custom solution here is tailored for fast configure times, but only
works for dependencies on Github.

* Import dependencies using the declared commands

* Remove git submodules

* Call target_link_libraries properly

target_link_libraries must not be called without a requirement
specifier.

* Fix includes for competition

Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>
This commit is contained in:
friendlyanon 2020-11-04 19:34:29 +01:00 committed by GitHub
parent 218c274090
commit c805fc28a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 187 additions and 202 deletions

42
.gitmodules vendored
View File

@ -1,42 +0,0 @@
[submodule "scalarvssimd/rapidjson"]
path = dependencies/rapidjson
url = https://github.com/Tencent/rapidjson.git
[submodule "dependencies/sajson"]
path = dependencies/sajson
url = https://github.com/chadaustin/sajson.git
[submodule "dependencies/json11"]
path = dependencies/json11
url = https://github.com/dropbox/json11.git
[submodule "dependencies/fastjson"]
path = dependencies/fastjson
url = https://github.com/mikeando/fastjson.git
[submodule "dependencies/gason"]
path = dependencies/gason
url = https://github.com/vivkin/gason.git
[submodule "dependencies/ujson4c"]
path = dependencies/ujson4c
url = https://github.com/esnme/ujson4c.git
[submodule "dependencies/jsmn"]
path = dependencies/jsmn
url = https://github.com/zserge/jsmn.git
[submodule "dependencies/cJSON"]
path = dependencies/cJSON
url = https://github.com/DaveGamble/cJSON.git
[submodule "dependencies/jsoncpp"]
path = dependencies/jsoncpp
url = https://github.com/open-source-parsers/jsoncpp.git
[submodule "dependencies/json"]
path = dependencies/json
url = https://github.com/nlohmann/json.git
[submodule "dependencies/benchmark"]
path = dependencies/benchmark
url = https://github.com/google/benchmark.git
[submodule "dependencies/cxxopts"]
path = dependencies/cxxopts
url = https://github.com/jarro2783/cxxopts
[submodule "dependencies/boost.json"]
path = dependencies/boost.json
url = https://github.com/CPPAlliance/json.git
[submodule "dependencies/yyjson"]
path = dependencies/yyjson
url = https://github.com/ibireme/yyjson.git

View File

@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.9) # CMP0069 NEW
cmake_minimum_required(VERSION 3.13)
project(simdjson
DESCRIPTION "Parsing gigabytes of JSON per second"

View File

@ -4,7 +4,7 @@ link_libraries(simdjson-windows-headers test-data)
if (TARGET benchmark::benchmark)
add_executable(bench_sax bench_sax.cpp)
target_link_libraries(bench_sax simdjson-internal-flags simdjson-include-source benchmark::benchmark)
target_link_libraries(bench_sax PRIVATE simdjson-internal-flags simdjson-include-source benchmark::benchmark)
endif (TARGET benchmark::benchmark)
link_libraries(simdjson simdjson-flags)
@ -24,19 +24,19 @@ target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARS
if (TARGET competition-all)
add_executable(distinctuseridcompetition distinctuseridcompetition.cpp)
target_link_libraries(distinctuseridcompetition competition-core)
target_link_libraries(distinctuseridcompetition PRIVATE competition-core)
add_executable(minifiercompetition minifiercompetition.cpp)
target_link_libraries(minifiercompetition competition-core)
target_link_libraries(minifiercompetition PRIVATE competition-core)
add_executable(parseandstatcompetition parseandstatcompetition.cpp)
target_link_libraries(parseandstatcompetition competition-core)
target_link_libraries(parseandstatcompetition PRIVATE competition-core)
add_executable(parsingcompetition parsingcompetition.cpp)
target_link_libraries(parsingcompetition competition-core)
target_link_libraries(parsingcompetition PRIVATE competition-core)
add_executable(allparsingcompetition parsingcompetition.cpp)
target_link_libraries(allparsingcompetition competition-all)
target_link_libraries(allparsingcompetition PRIVATE competition-all)
target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER)
endif()

View File

@ -7,7 +7,7 @@
# Clone the repository if it's not there
find_package(Git QUIET)
if (SIMDJSON_IS_UNDER_GIT AND SIMDJSON_GIT AND Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER "2.1.4") AND (NOT CMAKE_GENERATOR MATCHES Ninja) ) # We use "-C" which requires a recent git
if (Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER "2.1.4") AND (NOT CMAKE_GENERATOR MATCHES Ninja) ) # We use "-C" which requires a recent git
message(STATUS "Git is available and it is recent. We are enabling checkperf targets.")
# sync_git_repository(myrepo ...) creates two targets:
# myrepo - if the repo does not exist, creates and syncs it against the origin branch

View File

@ -40,21 +40,23 @@ using json = nlohmann::json;
#ifdef ALLPARSER
#include "fastjson.cpp"
#include "fastjson_dom.cpp"
#include "gason.cpp"
#include "fastjson/core.h"
#include "fastjson/dom.h"
#include "fastjson/fastjson.h"
#include "gason.h"
#include "json11.hpp"
#include "json11.cpp"
extern "C" {
#include "cJSON.c"
#include "cJSON.h"
#include "jsmn.c"
#include "jsmn.h"
#include "ujdecode.h"
#include "ultrajsondec.c"
extern "C" {
#include "ultrajson.h"
}
#include "jsoncpp.cpp"
#include "json/json.h"
#endif

View File

@ -9,14 +9,6 @@ else()
message (STATUS "By default, we just build the library.")
endif()
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git)
set(SIMDJSON_IS_UNDER_GIT ON CACHE BOOL "Whether cmake is under git control")
message( STATUS "The simdjson repository appears to be under git." )
else()
set(SIMDJSON_IS_UNDER_GIT OFF CACHE BOOL "Whether cmake is under git control")
message( STATUS "The simdjson repository does not appear to be under git." )
endif()
#
# Flags used by exes and by the simdjson library (project-wide flags)
#
@ -50,12 +42,6 @@ else()
option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
option(SIMDJSON_USE_LIBCPP "Use the libc++ library" OFF)
endif()
option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON)
option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" ON)
if(SIMDJSON_COMPETITION)
message(STATUS "Using SIMDJSON_GOOGLE_BENCHMARKS")
endif()
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake")
@ -185,8 +171,6 @@ endif(SIMDJSON_ONDEMAND_SAFETY_RAILS)
option(SIMDJSON_BASH "Allow usage of bash within CMake" ON)
option(SIMDJSON_GIT "Allow usage of git within CMake" ON)
option(SIMDJSON_EXCEPTIONS "Enable simdjson's exception-throwing interface" ON)
if(NOT SIMDJSON_EXCEPTIONS)
message(STATUS "simdjson exception interface turned off. Code that does not check error codes will not compile.")

View File

@ -1,106 +1,107 @@
# Initializes a git submodule if it hasn't been initialized before
include(import.cmake)
option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON)
option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" ON)
find_package(Git QUIET) # We want the library to build even if git is missing
if ((Git_FOUND) AND SIMDJSON_GIT AND (SIMDJSON_IS_UNDER_GIT))
message(STATUS "Git is available.")
# Does NOT attempt to update or otherwise modify git submodules that are already initialized.
function(initialize_submodule DIRECTORY)
if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/.git)
message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/.git does not exist. Initializing ${DIRECTORY} submodule ...")
execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
RESULT_VARIABLE GIT_EXIT_CODE)
if(NOT GIT_EXIT_CODE EQUAL "0")
message(FATAL_ERROR "${GIT_EXECUTABLE} submodule update --init dependencies/${DIRECTORY} failed with exit code ${GIT_EXIT_CODE}, please checkout submodules")
endif()
endif()
endfunction(initialize_submodule)
if(SIMDJSON_GOOGLE_BENCHMARKS)
set_off(BENCHMARK_ENABLE_TESTING)
set_off(BENCHMARK_ENABLE_INSTALL)
if (SIMDJSON_GOOGLE_BENCHMARKS)
message (STATUS "'SIMDJSON_GOOGLE_BENCHMARKS' is requested, configuring..." )
option(BENCHMARK_ENABLE_TESTING OFF)
set(BENCHMARK_ENABLE_TESTING OFF)
option(BENCHMARK_ENABLE_INSTALL OFF)
set(BENCHMARK_ENABLE_INSTALL OFF)
initialize_submodule(benchmark)
add_subdirectory(benchmark)
endif()
if (SIMDJSON_COMPETITION)
initialize_submodule(cJSON)
add_library(competition-cJSON INTERFACE)
target_include_directories(competition-cJSON INTERFACE cJSON)
initialize_submodule(fastjson)
add_library(competition-fastjson INTERFACE)
target_include_directories(competition-fastjson INTERFACE fastjson/src fastjson/include)
initialize_submodule(gason)
add_library(competition-gason INTERFACE)
target_include_directories(competition-gason INTERFACE gason/src)
initialize_submodule(jsmn)
add_library(competition-jsmn INTERFACE)
target_include_directories(competition-jsmn INTERFACE jsmn)
initialize_submodule(json)
add_library(competition-json INTERFACE)
target_include_directories(competition-json INTERFACE json/single_include)
initialize_submodule(json11)
add_library(competition-json11 INTERFACE)
target_include_directories(competition-json11 INTERFACE json11)
add_library(competition-jsoncppdist INTERFACE)
target_include_directories(competition-jsoncppdist INTERFACE jsoncppdist)
initialize_submodule(rapidjson)
add_library(competition-rapidjson INTERFACE)
target_include_directories(competition-rapidjson INTERFACE rapidjson/include)
initialize_submodule(sajson)
add_library(competition-sajson INTERFACE)
target_include_directories(competition-sajson INTERFACE sajson/include)
initialize_submodule(ujson4c)
add_library(competition-ujson4c ujson4c/src/ujdecode.c)
target_include_directories(competition-ujson4c PUBLIC ujson4c/3rdparty ujson4c/src)
# Boost JSON is not compatible with some clang/libc++ configurations.
if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
initialize_submodule(boost.json)
add_library(boostjson boost.json/src/src.cpp)
target_compile_definitions(boostjson PUBLIC BOOST_JSON_STANDALONE)
target_include_directories(boostjson PUBLIC boost.json/include)
endif()
initialize_submodule(yyjson)
add_library(yyjson yyjson/src/yyjson.c)
target_include_directories(yyjson PUBLIC yyjson/src)
add_library(competition-core INTERFACE)
# Boost JSON is not compatible with some clang/libc++ configurations.
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
target_link_libraries(competition-core INTERFACE competition-json competition-rapidjson competition-sajson competition-cJSON competition-jsmn yyjson)
else()
target_link_libraries(competition-core INTERFACE competition-json competition-rapidjson competition-sajson competition-cJSON competition-jsmn boostjson yyjson)
endif()
add_library(competition-all INTERFACE)
target_link_libraries(competition-all INTERFACE competition-core competition-jsoncppdist competition-json11 competition-fastjson competition-gason competition-ujson4c)
endif()
initialize_submodule(cxxopts)
message(STATUS "We acquired cxxopts and we are adding it as a library and target.")
add_library(cxxopts INTERFACE)
target_include_directories(cxxopts INTERFACE cxxopts/include)
else()
message(STATUS "Git is unavailable.")
if(SIMDJSON_COMPETITION)
message (STATUS "'SIMDJSON_COMPETITION' is requested, but we cannot download the remote repositories." )
endif()
if(SIMDJSON_GOOGLE_BENCHMARKS)
message (STATUS "'SIMDJSON_GOOGLE_BENCHMARKS' is requested, but we cannot download the remote repositories." )
endif()
import_dependency(google_benchmarks google/benchmark 8982e1e)
add_dependency(google_benchmarks)
endif()
# This prevents variables declared with set() from unnecessarily escaping and
# should not be called more than once
function(competition_scope_)
# Boost JSON is not compatible with some clang/libc++ configurations.
if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
import_dependency(boostjson CPPAlliance/json a0983f7)
add_library(boostjson STATIC "${boostjson_SOURCE_DIR}/src/src.cpp")
target_compile_definitions(boostjson PUBLIC BOOST_JSON_STANDALONE)
target_include_directories(boostjson SYSTEM PUBLIC
"${boostjson_SOURCE_DIR}/include")
endif()
import_dependency(cjson DaveGamble/cJSON c69134d)
add_library(cjson STATIC "${cjson_SOURCE_DIR}/cJSON.c")
target_include_directories(cjson SYSTEM PUBLIC "${cjson_SOURCE_DIR}")
import_dependency(fastjson mikeando/fastjson 485f994)
add_library(fastjson STATIC
"${fastjson_SOURCE_DIR}/src/fastjson.cpp"
"${fastjson_SOURCE_DIR}/src/fastjson2.cpp"
"${fastjson_SOURCE_DIR}/src/fastjson_dom.cpp")
target_include_directories(fastjson SYSTEM PUBLIC
"${fastjson_SOURCE_DIR}/include")
import_dependency(gason vivkin/gason 7aee524)
add_library(gason STATIC "${gason_SOURCE_DIR}/src/gason.cpp")
target_include_directories(gason SYSTEM PUBLIC "${gason_SOURCE_DIR}/src")
import_dependency(jsmn zserge/jsmn 18e9fe4)
add_library(jsmn STATIC "${jsmn_SOURCE_DIR}/jsmn.c")
target_include_directories(jsmn SYSTEM PUBLIC "${jsmn_SOURCE_DIR}")
message(STATUS "Importing json (nlohmann/json@v3.9.1)")
set(nlohmann_json_SOURCE_DIR "${dep_root}/json")
if(NOT EXISTS "${nlohmann_json_SOURCE_DIR}")
file(DOWNLOAD
"https://github.com/nlohmann/json/releases/download/v3.9.1/json.hpp"
"${nlohmann_json_SOURCE_DIR}/nlohmann/json.hpp")
endif()
add_library(nlohmann_json INTERFACE)
target_include_directories(nlohmann_json SYSTEM INTERFACE "${nlohmann_json_SOURCE_DIR}")
import_dependency(json11 dropbox/json11 ec4e452)
add_library(json11 STATIC "${json11_SOURCE_DIR}/json11.cpp")
target_include_directories(json11 SYSTEM PUBLIC "${json11_SOURCE_DIR}")
set(jsoncpp_SOURCE_DIR "${simdjson_SOURCE_DIR}/dependencies/jsoncppdist")
add_library(jsoncpp STATIC "${jsoncpp_SOURCE_DIR}/jsoncpp.cpp")
target_include_directories(jsoncpp SYSTEM PUBLIC "${jsoncpp_SOURCE_DIR}")
import_dependency(rapidjson Tencent/rapidjson b32cd94)
add_library(rapidjson INTERFACE)
target_compile_definitions(rapidjson INTERFACE RAPIDJSON_HAS_STDSTRING)
target_include_directories(rapidjson SYSTEM INTERFACE
"${rapidjson_SOURCE_DIR}/include")
import_dependency(sajson chadaustin/sajson 2dcfd35)
add_library(sajson INTERFACE)
target_compile_definitions(sajson INTERFACE SAJSON_UNSORTED_OBJECT_KEYS)
target_include_directories(sajson SYSTEM INTERFACE
"${sajson_SOURCE_DIR}/include")
import_dependency(ujson4c esnme/ujson4c e14f3fd)
add_library(ujson4c STATIC
"${ujson4c_SOURCE_DIR}/src/ujdecode.c"
"${ujson4c_SOURCE_DIR}/3rdparty/ultrajsondec.c")
target_include_directories(ujson4c SYSTEM PUBLIC
"${ujson4c_SOURCE_DIR}/src"
"${ujson4c_SOURCE_DIR}/3rdparty")
import_dependency(yyjson ibireme/yyjson aa33ec5)
add_library(yyjson STATIC "${yyjson_SOURCE_DIR}/src/yyjson.c")
target_include_directories(yyjson SYSTEM PUBLIC "${yyjson_SOURCE_DIR}/src")
add_library(competition-core INTERFACE)
target_link_libraries(competition-core INTERFACE nlohmann_json rapidjson sajson cjson jsmn yyjson)
if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
target_link_libraries(competition-core INTERFACE boostjson)
endif()
add_library(competition-all INTERFACE)
target_link_libraries(competition-all INTERFACE competition-core jsoncpp json11 fastjson gason ujson4c)
endfunction()
if(SIMDJSON_COMPETITION)
competition_scope_()
endif()
set_off(CXXOPTS_BUILD_EXAMPLES)
set_off(CXXOPTS_BUILD_TESTS)
set_off(CXXOPTS_ENABLE_INSTALL)
import_dependency(cxxopts jarro2783/cxxopts 794c975)
add_dependency(cxxopts)

@ -1 +0,0 @@
Subproject commit 8982e1ee6aef31e48170400b7d1dc9969b156e5e

@ -1 +0,0 @@
Subproject commit a0983f788b9138211bfc060e68c9973efb3394e4

1
dependencies/cJSON vendored

@ -1 +0,0 @@
Subproject commit c69134d01746dcf551dd7724b4edb12f922eb0d1

@ -1 +0,0 @@
Subproject commit 794c975287355de48158d9a80ed502d26b20a472

@ -1 +0,0 @@
Subproject commit 485f994a61a64ac73fa6a40d4d639b99b463563b

1
dependencies/gason vendored

@ -1 +0,0 @@
Subproject commit 7aee524189da1c1ecd19f67981e3d903dae25470

48
dependencies/import.cmake vendored Normal file
View File

@ -0,0 +1,48 @@
set(dep_root "${simdjson_SOURCE_DIR}/dependencies/.cache")
if(DEFINED ENV{simdjson_DEPENDENCY_CACHE_DIR})
set(dep_root "$ENV{simdjson_DEPENDENCY_CACHE_DIR}")
endif()
function(import_dependency NAME GITHUB_REPO COMMIT)
message(STATUS "Importing ${NAME} (${GITHUB_REPO}@${COMMIT})")
set(target "${dep_root}/${NAME}")
# If the folder exists in the cache, then we assume that everything is as
# should be and do nothing
if(EXISTS "${target}")
set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
return()
endif()
set(zip_url "https://github.com/${GITHUB_REPO}/archive/${COMMIT}.zip")
set(archive "${dep_root}/archive.zip")
set(dest "${dep_root}/_extract")
file(DOWNLOAD "${zip_url}" "${archive}")
file(MAKE_DIRECTORY "${dest}")
execute_process(
WORKING_DIRECTORY "${dest}"
COMMAND "${CMAKE_COMMAND}" -E tar xf "${archive}")
file(REMOVE "${archive}")
# GitHub archives only ever have one folder component at the root, so this
# will always match that single folder
file(GLOB dir LIST_DIRECTORIES YES "${dest}/*")
file(RENAME "${dir}" "${target}")
set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
endfunction()
# Delegates to the dependency
macro(add_dependency NAME)
if(NOT DEFINED "${NAME}_SOURCE_DIR")
message(FATAL_ERROR "Missing ${NAME}_SOURCE_DIR variable")
endif()
add_subdirectory("${${NAME}_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/_deps/${NAME}")
endmacro()
function(set_off NAME)
set("${NAME}" OFF CACHE INTERNAL "")
endfunction()

1
dependencies/jsmn vendored

@ -1 +0,0 @@
Subproject commit 18e9fe42cbfe21d65076f5c77ae2be379ad1270f

1
dependencies/json vendored

@ -1 +0,0 @@
Subproject commit a015b78e81c859b88840cb0cd4001ce1fe5e7865

1
dependencies/json11 vendored

@ -1 +0,0 @@
Subproject commit ec4e45219af1d7cde3d58b49ed762376fccf1ace

@ -1 +0,0 @@
Subproject commit 0c1cc6e1a373dc58e2599ec7dd68b2e6b863990a

@ -1 +0,0 @@
Subproject commit b32cd9421c5e3cbe183a99b6537ce11441e50656

1
dependencies/sajson vendored

@ -1 +0,0 @@
Subproject commit 2dcfd350586375f9910f74821d4f07d67ae455ba

@ -1 +0,0 @@
Subproject commit e14f3fd5207fe30d1bdea723f260609e69d1abfa

1
dependencies/yyjson vendored

@ -1 +0,0 @@
Subproject commit aa33ec5a4769ce38078a5bdeecbdacfdf1cad8fb

View File

@ -71,7 +71,7 @@ add_cpp_test(padded_string_tests)
find_program(BASH bash)
# Below we skip anything on Windows, not just visual studio, because running bash under Windows requires you to
# Below we skip anything on Windows, not just visual studio, because running bash under Windows requires you to
# map app paths to their "mounted" equivalent (e.g., /mnt/c/...). So even if you have bash under Windows, extra work would be
# required to make things work robustly. Simply put: bash is not quite portable.
@ -94,7 +94,7 @@ if (BASH AND (NOT WIN32) AND SIMDJSON_BASH AND (TARGET json2json)) # The scripts
if ((SIMDJSON_COMPETITION) AND (!SIMDJSON_SANITIZE))
# It looks like RapidJSON does not pass the sanitizer under some conditions (Clang 10)
add_executable(allparserscheckfile allparserscheckfile.cpp)
target_link_libraries(allparserscheckfile competition-all)
target_link_libraries(allparserscheckfile PRIVATE competition-all)
add_test(issue150 ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
set_property(TEST issue150 APPEND PROPERTY DEPENDS allparserscheckfile)

View File

@ -6,24 +6,30 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
// #define RAPIDJSON_SSE2 // bad
// #define RAPIDJSON_SSE42 // bad
#include "fastjson.cpp"
#include "fastjson_dom.cpp"
#include "gason.cpp"
#include "json11.cpp"
#include "fastjson/core.h"
#include "fastjson/dom.h"
#include "fastjson/fastjson.h"
#include "gason.h"
#include "json11.hpp"
#include "rapidjson/document.h"
#include "rapidjson/reader.h" // you have to check in the submodule
#include "rapidjson/stringbuffer.h"
#include "rapidjson/writer.h"
#include "sajson.h"
extern "C" {
#include "cJSON.c"
#include "cJSON.h"
#include "jsmn.c"
#include "jsmn.h"
#include "ujdecode.h"
#include "ultrajsondec.c"
extern "C" {
#include "ultrajson.h"
}
#include "jsoncpp.cpp"
#include "json/json.h"
SIMDJSON_POP_DISABLE_WARNINGS