Merge pull request #727 from simdjson/jkeiser/cmake-checkperf

Add checkperf to cmake

.circleci/config.yml

@@ -8,7 +8,7 @@ executors:
         environment:
           CXX: g++
           BUILD_FLAGS: -j
-          CTEST_FLAGS: -j --output-on-failure
+          CTEST_FLAGS: -j4 --output-on-failure
 
   gcc8:
     docker:
@@ -16,7 +16,7 @@ executors:
         environment:
           CXX: g++
           BUILD_FLAGS: -j
-          CTEST_FLAGS: -j --output-on-failure
+          CTEST_FLAGS: -j4 --output-on-failure
 
   gcc9:
     docker:
@@ -24,7 +24,7 @@ executors:
         environment:
           CXX: g++
           BUILD_FLAGS: -j
-          CTEST_FLAGS: -j --output-on-failure
+          CTEST_FLAGS: -j4 --output-on-failure
 
   clang6:
     docker:
@@ -32,7 +32,7 @@ executors:
         environment:
           CXX: clang++-6.0
           BUILD_FLAGS: -j
-          CTEST_FLAGS: -j --output-on-failure
+          CTEST_FLAGS: -j4 --output-on-failure
                 
 # Reusable test commands (and initializer for clang 6)
 commands:
@@ -67,7 +67,7 @@ commands:
     steps:
       - cmake_build
       - run: ctest $CTEST_FLAGS -L acceptance
-      - run: ctest $CTEST_FLAGS -LE acceptance
+      - run: ctest $CTEST_FLAGS -LE acceptance -E checkperf
 
   cmake_test_all:
     steps:
@@ -99,20 +99,15 @@ jobs:
     steps: [ init_clang6, cmake_test_all, cmake_install_test ]
   
   # sanitize
-  sanitize-gcc7:
-    description: Build and run tests on GCC 7 and AVX 2 with a cmake sanitize build
-    executor: gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "" } # gcc sanitize seems to have bugs with -j :(
-    steps: [ cmake_test_all ]
   sanitize-gcc9:
     description: Build and run tests on GCC 9 and AVX 2 with a cmake sanitize build
     executor: gcc9
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
     steps: [ cmake_test_all ]
   sanitize-clang6:
     description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build
     executor: clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
     steps: [ init_clang6, cmake_test_all ]
   
   # dynamic
@@ -181,7 +176,6 @@ workflows:
       - clang6
 
       # full single-implementation tests
-      - sanitize-gcc7
       - sanitize-gcc9
       - sanitize-clang6
       - dynamic-gcc7

.drone.yml

@@ -231,7 +231,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: amd64_clang_cmake_no_exceptions
@@ -254,7 +254,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: amd64_clang_cmake_static
@@ -277,7 +277,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: amd64_gcc_cmake_static
@@ -300,7 +300,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: amd64_gcc_cmake_dynamic
@@ -323,7 +323,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: arm64_clang_cmake_dynamic
@@ -346,7 +346,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: arm64_gcc_cmake_dynamic
@@ -369,7 +369,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: arm64_clang_cmake_static
@@ -392,7 +392,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: arm64_gcc_cmake_static
@@ -415,7 +415,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ctest -j --output-on-failure
+    - ctest -j4 --output-on-failure -E checkperf
 
 ---
 kind: pipeline
@@ -439,7 +439,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ASAN_OPTIONS="detect_leaks=0" ctest -j --output-on-failure
+    - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: arm64_gcc_cmake_sanitize
@@ -462,7 +462,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ASAN_OPTIONS="detect_leaks=0" ctest -j --output-on-failure
+    - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf
 
 ---
 kind: pipeline
@@ -486,7 +486,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ASAN_OPTIONS="detect_leaks=0" ctest -j --output-on-failure
+    - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: amd64_gcc_cmake_sanitize
@@ -509,7 +509,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ASAN_OPTIONS="detect_leaks=0" ctest -j --output-on-failure
+    - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf
 
 ---
 kind: pipeline
@@ -533,7 +533,7 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ASAN_OPTIONS="detect_leaks=0" ctest -j --output-on-failure
+    - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf
 ---
 kind: pipeline
 name: amd64_gcc_cmake_sanitize_noavx
@@ -556,4 +556,4 @@ steps:
     - mkdir build && cd build
     - cmake $CMAKE_FLAGS ..
     - make -j
-    - ASAN_OPTIONS="detect_leaks=0" ctest -j --output-on-failure
+    - ASAN_OPTIONS="detect_leaks=0" ctest -j4 --output-on-failure -E checkperf

.gitignore

@@ -53,6 +53,7 @@ libs
 objs
 
 # Build outputs (TODO build to a subdir so we can exclude that instead)
+/.simdjson-user-CMakeCache.txt
 /allparserscheckfile
 /allparsingcompetition
 /basictests
@@ -71,6 +72,7 @@ objs
 /benchmark/parse_stream
 /benchmark/parsingcompetition
 /benchmark/perfdiff
+/benchmark/simdjson-master
 /benchmark/statisticalmodel
 /build/
 /build-ossfuzz-*/
@@ -123,14 +125,15 @@ objs
 /stringparsingcheck
 /submodules
 /ujdecode.o
-/amalgamation_demo.cpp
+/amalgamate_demo.cpp
 /simdjson.cpp
 /simdjson.h
-/singleheader/amalgamation_demo
+/singleheader/amalgamate_demo
 /singleheader/demo
 /tests/allparserscheckfile
 /tests/basictests
 /tests/checkimplementation
+/tests/compilation_failure_tests/example_compiletest_should_compile
 /tests/errortests
 /tests/extracting_values_example
 /tests/integer_tests

CMakeLists.txt

@@ -17,6 +17,7 @@ set(PROJECT_VERSION_MINOR 3)
 set(PROJECT_VERSION_PATCH 1)
 set(SIMDJSON_LIB_VERSION "0.3.1" CACHE STRING "simdjson library version")
 set(SIMDJSON_LIB_SOVERSION "1" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_GITHUB_REPOSITORY https://github.com/simdjson/simdjson)
 
 if(MSVC)
   option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library
@@ -120,6 +121,23 @@ endfunction()
 
 export_private_library(simdjson-flags)
 
+#
+# ${SIMDJSON_USER_CMAKECACHE} contains the *user-specified* simdjson options so you can call cmake on
+# another branch or repository with the same options.
+#
+# Not supported on Windows at present, because the only thing that uses it is checkperf, which we
+# don't run on Windows.
+#
+if (NOT MSVC)
+  set(SIMDJSON_USER_CMAKECACHE ${CMAKE_CURRENT_BINARY_DIR}/.simdjson-user-CMakeCache.txt)
+  add_custom_command(
+    OUTPUT ${SIMDJSON_USER_CMAKECACHE}
+    COMMAND bash -c "grep SIMDJSON_ ${PROJECT_BINARY_DIR}/CMakeCache.txt | grep -v SIMDJSON_LIB_ > ${SIMDJSON_USER_CMAKECACHE}"
+    VERBATIM # Makes it not do weird escaping with the command
+  )
+  add_custom_target(simdjson-user-cmakecache ALL DEPENDS ${SIMDJSON_USER_CMAKECACHE})
+endif()
+
 #
 # Create the top level simdjson library (must be done at this level to use both src/ and include/
 # directories) and tools

Makefile

@@ -1,4 +1,4 @@
-REFERENCE_VERSION = master
+CHECKPERF_BRANCH = master
 
 .SUFFIXES:
 #
@@ -207,8 +207,8 @@ benchfeatures: benchmark/benchfeatures.cpp benchmark/event_counter.h benchmark/b
 perfdiff: benchmark/perfdiff.cpp
 	$(CXX) $(CXXFLAGS) -o perfdiff benchmark/perfdiff.cpp $(LIBFILES) $(LIBFLAGS)
 
-checkperf:
+checkperf: parse perfdiff
-	bash ./scripts/checkperf.sh $(REFERENCE_VERSION)
+	CHECKPERF_BRANCH=$(CHECKPERF_BRANCH) bash ./benchmark/checkperf.sh jsonexamples/twitter.json
 
 statisticalmodel: benchmark/statisticalmodel.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o statisticalmodel benchmark/statisticalmodel.cpp $(LIBFILES) $(LIBFLAGS)

benchmark/CMakeLists.txt

@@ -34,3 +34,16 @@ if (SIMDJSON_COMPETITION)
   target_link_libraries(allparsingcompetition competition-all)
   target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER)
 endif()
+
+if (NOT MSVC)
+  add_test(NAME checkperf
+    COMMAND ${CMAKE_COMMAND} -E env
+      CHECKPERF_REPOSITORY=https://github.com/simdjson/simdjson
+      CHECKPERF_BRANCH=master
+      CHECKPERF_DIR=${CMAKE_CURRENT_BINARY_DIR}/simdjson-master
+      CHECKPERF_CMAKECACHE=${SIMDJSON_USER_CMAKECACHE}
+      bash ${CMAKE_CURRENT_SOURCE_DIR}/checkperf.sh ${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json)
+  set_property(TEST checkperf APPEND PROPERTY LABELS per_implementation)
+  set_property(TEST checkperf APPEND PROPERTY DEPENDS parse perfdiff ${SIMDJSON_USER_CMAKECACHE})
+  set_property(TEST checkperf PROPERTY RUN_SERIAL TRUE)
+endif()

benchmark/checkperf.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -ex
+
+set | grep CHECKPERF
+
+if [ -z "$CHECKPERF_REPOSITORY" ]; then CHECKPERF_REPOSITORY=.; fi
+
+# Arguments: perfdiff.sh <branch> <test json files>
+if [ -z "$CHECKPERF_BRANCH" ]; then CHECKPERF_BRANCH=master; fi
+if [ -z "$CHECKPERF_DIR" ]; then CHECKPERF_DIR=benchbranch/$CHECKPERF_BRANCH; fi
+if [ -z "$CHECKPERF_ARGS" ]; then
+  if [ -z "$*" ]; then
+    CHECKPERF_ARGS=jsonexamples/twitter.json;
+  else
+    CHECKPERF_ARGS=$*;
+  fi
+fi
+
+# Clone and build the reference branch's parse
+if [ -d $CHECKPERF_DIR/.git ]; then
+  echo "Checking out the reference branch ($CHECKPERF_BRANCH) into $CHECKPERF_DIR ..."
+  pushd $CHECKPERF_DIR
+  git remote update
+  git reset --hard origin/$CHECKPERF_BRANCH
+else
+  echo "Cloning the reference branch ($CHECKPERF_BRANCH) into $CHECKPERF_DIR ..."
+  mkdir -p $CHECKPERF_DIR
+  git clone --depth 1 -b $CHECKPERF_BRANCH $CHECKPERF_REPOSITORY $CHECKPERF_DIR
+  pushd $CHECKPERF_DIR
+fi
+
+echo "Building $CHECKPERF_DIR/parse ..."
+if [ -n "$CHECKPERF_CMAKECACHE" ]; then
+  cp $CHECKPERF_CMAKECACHE $CHECKPERF_DIR/CMakeCache.txt
+  cmake -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_COMPETITION=OFF .
+  make parse
+  REFERENCE_PARSE=$CHECKPERF_DIR/benchmark/parse
+else
+  make parse
+  REFERENCE_PARSE=$CHECKPERF_DIR/parse
+fi
+popd
+
+# Run them and diff performance
+echo "Running perfdiff:"
+echo ./perfdiff \"./parse -t $CHECKPERF_ARGS\" \"$REFERENCE_PARSE -t $CHECKPERF_ARGS\"
+./perfdiff "./parse -t $CHECKPERF_ARGS" "$REFERENCE_PARSE -t $CHECKPERF_ARGS"

scripts/checkperf.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-set -e
-SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
-
-if [ -z "$CHECKPERF_REPOSITORY" ]; then CHECKPERF_REPOSITORY=.; fi
-
-# Arguments: perfdiff.sh <branch> <test json files>
-if [ -z "$1" ]; then reference_branch="master"; else reference_branch=$1; shift; fi
-if [ -z "$*" ]; then perftests="jsonexamples/twitter.json"; else perftests=$*; fi
-
-# Clone and build the reference branch's parse
-echo "Cloning and build the reference branch ($reference_branch) ..."
-current=$SCRIPTPATH/..
-reference=$current/benchbranch/$reference_branch
-rm -rf $reference
-mkdir -p $reference
-git clone --depth 1 -b $reference_branch $CHECKPERF_REPOSITORY $reference
-cd $reference
-make parse
-
-# Build the current branch's parse
-echo "Building the current branch ..."
-cd $current
-make clean
-make parse
-
-# Run them and diff performance
-make perfdiff
-
-echo "Running perfdiff:"
-echo ./perfdiff \"$current/parse -t $perftests $CHECKPERF_ARGS\" \"$reference/parse -t $perftests $CHECKPERF_ARGS\"
-./perfdiff "$current/parse -t $perftests $CHECKPERF_ARGS" "$reference/parse -t $perftests $CHECKPERF_ARGS"

src/CMakeLists.txt

@@ -49,8 +49,8 @@ target_link_libraries(simdjson PRIVATE simdjson-source simdjson-flags)
 if(NOT MSVC)
   ## We output the library at the root of the current directory where cmake is invoked
   ## This is handy but Visual Studio will happily ignore us
-  set_target_properties(simdjson PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+  set_target_properties(simdjson PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
-  MESSAGE( STATUS "Library output directory (does not apply to Visual Studio): " ${CMAKE_BINARY_DIR})
+  MESSAGE( STATUS "Library output directory (does not apply to Visual Studio): " ${PROJECT_BINARY_DIR})
 endif()
 
 #