Add Google Benchmark for calling conventions

- disable it on ubuntu 18.04 tests, which fail for [really can't figure
out why]

.appveyor.yml

@@ -23,7 +23,7 @@ environment:
 build_script:
   - mkdir build
   - cd build
-  - ps: cmake -DSIMDJSON_BUILD_STATIC="$env:SIMDJSON_BUILD_STATIC" -DSIMDJSON_ENABLE_THREADS="$env:THREADS" -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=x64 ..
+  - ps: cmake -DSIMDJSON_BUILD_STATIC="$env:SIMDJSON_BUILD_STATIC" -DSIMDJSON_ENABLE_THREADS="$env:THREADS" -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_GOOGLE_BENCHMARKS=OFF ..
   - cmake --build .
   - ctest --verbose --output-on-failure
 

.drone.yml

@@ -126,12 +126,13 @@ steps:
   environment:
     CC: clang
     CXX: clang++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y clang make cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 ---
@@ -148,12 +149,13 @@ steps:
   environment:
     CC: clang
     CXX: clang++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y clang make cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 ---
@@ -170,12 +172,13 @@ steps:
   environment:
     CC: gcc
     CXX: g++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
   commands:
     - apt-get update -qq
     - apt-get install -y cmake 
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 ---
@@ -192,12 +195,13 @@ steps:
   environment:
     CC: gcc
     CXX: g++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 ---
@@ -214,12 +218,13 @@ steps:
   environment:
     CC: clang
     CXX: clang++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y clang make cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 ---
@@ -236,12 +241,13 @@ steps:
   environment:
     CC: gcc
     CXX: g++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 ---
@@ -258,12 +264,13 @@ steps:
   environment:
     CC: clang
     CXX: clang++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y clang make cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 ---
@@ -280,12 +287,13 @@ steps:
   environment:
     CC: gcc
     CXX: g++
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
   commands:
     - apt-get update -qq
     - apt-get install -y cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ctest --output-on-failure
 
@@ -303,12 +311,13 @@ steps:
   environment:
     CC: clang
     CXX: clang++
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y clang make cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
 ---
@@ -325,12 +334,13 @@ steps:
   environment:
     CC: gcc
     CXX: g++
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
 
@@ -348,12 +358,13 @@ steps:
   environment:
     CC: clang
     CXX: clang++
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -y
     - apt-get install -y make $CC g++ cmake 
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
 ---
@@ -370,12 +381,13 @@ steps:
   environment:
     CC: gcc
     CXX: g++
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
 
@@ -393,12 +405,13 @@ steps:
   environment:
     CC: clang
     CXX: clang++
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y clang make cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_SANITIZE=ON  -DSIMDJSON_DISABLE_AVX=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
 ---
@@ -415,11 +428,12 @@ steps:
   environment:
     CC: gcc
     CXX: g++
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_GOOGLE_BENCHMARKS=OFF
   commands:
     - apt-get update -qq
     - apt-get install -y cmake
     - $CC --version
     - mkdir build && cd build
-    - cmake -DSIMDJSON_SANITIZE=ON -DSIMDJSON_DISABLE_AVX=ON $CMAKE_FLAGS ..
+    - cmake $CMAKE_FLAGS ..
     - make -j
     - ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure

.gitignore

@@ -57,6 +57,8 @@ objs
 /allparsingcompetition
 /basictests
 /benchfeatures
+/benchmark/bench_parse_call
+/benchmark/get_corpus_benchmark
 /benchmark/parse
 /benchmark/parse_stream
 /benchmark/perfdiff
@@ -67,7 +69,9 @@ objs
 /corpus.zip
 /distinctuseridcompetition
 /fuzz/fuzz_dump
+/fuzz/fuzz_dump_raw_tape
 /fuzz/fuzz_parser
+/fuzz/fuzz_print_json
 /get_corpus_benchmark
 /json2json
 /jsoncheck

.gitmodules

@@ -28,3 +28,9 @@
 [submodule "dependencies/json"]
 	path = dependencies/json
 	url = https://github.com/nlohmann/json.git
+[submodule "dependencies/benchmark"]
+	path = dependencies/benchmark
+	url = https://github.com/google/benchmark.git
+[submodule "dependencies/googletest"]
+	path = dependencies/googletest
+	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -1,8 +1,8 @@
 cmake_minimum_required(VERSION 3.9) # CMP0069 NEW
 
 if (NOT CMAKE_BUILD_TYPE)
-                message(STATUS "No build type selected, default to Release")
+  message(STATUS "No build type selected, default to Release")
-                set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
+  set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
 endif()
 
 project(simdjson 
@@ -15,7 +15,7 @@ project(simdjson
 #include(CheckIPOSupported)
 #check_ipo_supported(RESULT ltoresult)
 #if(ltoresult)
-#set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+#  set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
 #endif()
 
 # usage: cmake -DSIMDJSON_DISABLE_AVX=on ..
@@ -37,9 +37,9 @@ set(SIMDJSON_LIB_VERSION "0.2.1" CACHE STRING "simdjson library version")
 set(SIMDJSON_LIB_SOVERSION "0" CACHE STRING "simdjson library soversion")
 
 if(NOT MSVC)
-option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
+  option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
 else()
-option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library
+  option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library
 endif()
 option(SIMDJSON_SANITIZE "Sanitize addresses" OFF)
 
@@ -49,7 +49,6 @@ find_package(CTargets)
 find_package(Options)
 
 option(SIMDJSON_ENABLE_THREADS "enable threaded operation" ON)
-
 install(DIRECTORY include/${SIMDJSON_LIB_NAME} DESTINATION include)
 set (TEST_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/")
 set (BENCHMARK_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/")
@@ -62,10 +61,28 @@ add_subdirectory(tools)
 add_subdirectory(tests)
 add_subdirectory(benchmark)
 
+option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" OFF)
+if (SIMDJSON_GOOGLE_BENCHMARKS)
+  set(BENCHMARK_ENABLE_GTEST_TESTS OFF)
+  if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dependencies/googletest/CMakeLists.txt)
+    # message(STATUS "Unable to find ${CMAKE_CURRENT_SOURCE_DIR}/dependencies/googletest/CMakeLists.txt")
+    execute_process(COMMAND git submodule update --init -- ${CMAKE_CURRENT_SOURCE_DIR}/dependencies/googletest
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+  if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dependencies/benchmark/CMakeLists.txt)
+    # message(STATUS "Unable to find ${CMAKE_CURRENT_SOURCE_DIR}/dependencies/benchmark/CMakeLists.txt")
+    execute_process(COMMAND git submodule update --init -- ${CMAKE_CURRENT_SOURCE_DIR}/dependencies/benchmark
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/dependencies/googletest)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/dependencies/benchmark)
+endif()
+
 # for fuzzing, read the comments in the fuzz/CMakeLists.txt file
 option(ENABLE_FUZZING "enable building the fuzzers" ON)
 if(ENABLE_FUZZING)
- add_subdirectory(fuzz)
+  add_subdirectory(fuzz)
 endif()
 
 set(CPACK_PACKAGE_VENDOR "Daniel Lemire")

benchmark/CMakeLists.txt

@@ -8,5 +8,10 @@ add_cpp_benchmark(parse)
 add_cpp_benchmark(statisticalmodel)
 add_cpp_benchmark(parse_stream)
 add_cpp_benchmark(get_corpus_benchmark)
-
 add_executable(perfdiff perfdiff.cpp)
+
+# Google Benchmarks
+if (SIMDJSON_GOOGLE_BENCHMARKS)
+  add_cpp_benchmark(bench_parse_call bench_parse_call.cpp)
+  target_link_libraries(bench_parse_call benchmark::benchmark)
+endif()

benchmark/bench_parse_call.cpp

@@ -0,0 +1,66 @@
+#include <benchmark/benchmark.h>
+#include "simdjson/document.h"
+#include "simdjson/jsonparser.h"
+using namespace simdjson;
+using namespace benchmark;
+using namespace std;
+
+const padded_string EMPTY_ARRAY("[]", 2);
+
+static void json_parse(State& state) {
+  document::parser parser;
+  if (!parser.allocate_capacity(EMPTY_ARRAY.length())) { return; }
+  for (auto _ : state) {
+    auto error = simdjson::json_parse(EMPTY_ARRAY, parser);
+    if (error) { return; }
+  }
+}
+BENCHMARK(json_parse);
+static void parser_parse_error_code(State& state) {
+  document::parser parser;
+  if (!parser.allocate_capacity(EMPTY_ARRAY.length())) { return; }
+  for (auto _ : state) {
+    auto [doc, error] = parser.parse(EMPTY_ARRAY);
+    if (error) { return; }
+  }
+}
+BENCHMARK(parser_parse_error_code);
+static void parser_parse_exception(State& state) {
+  document::parser parser;
+  if (!parser.allocate_capacity(EMPTY_ARRAY.length())) { return; }
+  for (auto _ : state) {
+    try {
+      UNUSED document &doc = parser.parse(EMPTY_ARRAY);
+    } catch(invalid_json &j) {
+      return;
+    }
+  }
+}
+BENCHMARK(parser_parse_exception);
+
+static void build_parsed_json(State& state) {
+  for (auto _ : state) {
+    document::parser parser = simdjson::build_parsed_json(EMPTY_ARRAY);
+    if (!parser.valid) { return; }
+  }
+}
+BENCHMARK(build_parsed_json);
+static void document_parse_error_code(State& state) {
+  for (auto _ : state) {
+    auto [doc, error] = document::parse(EMPTY_ARRAY);
+    if (error) { return; }
+  }
+}
+BENCHMARK(document_parse_error_code);
+static void document_parse_exception(State& state) {
+  for (auto _ : state) {
+    try {
+      UNUSED document doc = document::parse(EMPTY_ARRAY);
+    } catch(invalid_json &j) {
+      return;
+    }
+  }
+}
+BENCHMARK(document_parse_exception);
+
+BENCHMARK_MAIN();

dependencies/benchmark

@@ -0,0 +1 @@
+Subproject commit 8982e1ee6aef31e48170400b7d1dc9969b156e5e

dependencies/googletest

@@ -0,0 +1 @@
+Subproject commit 23b2a3b1cf803999fb38175f6e9e038a4495c8a5