Consolidate multi-implementation tests

Uses SIMDJSON_FORCE_IMPLEMENTATION to switch the implementation at test
time.

.circleci/config.yml

@@ -7,72 +7,24 @@ executors:
       - image: gcc:7
         environment:
           CXX: g++
-  haswell-gcc7:
-    docker:
-      - image: gcc:7
-        environment:
-          CXX: g++
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=ON  -DSIMDJSON_IMPLEMENTATION_WESTMERE=OFF -DSIMDJSON_IMPLEMENTATION_FALLBACK=OFF -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
-  westmere-gcc7:
-    docker:
-      - image: gcc:7
-        environment:
-          CXX: g++
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=OFF -DSIMDJSON_IMPLEMENTATION_WESTMERE=ON  -DSIMDJSON_IMPLEMENTATION_FALLBACK=OFF -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
-  fallback-gcc7:
-    docker:
-      - image: gcc:7
-        environment:
-          CXX: g++
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=OFF -DSIMDJSON_IMPLEMENTATION_WESTMERE=OFF -DSIMDJSON_IMPLEMENTATION_FALLBACK=ON  -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
 
   gcc8:
     docker:
       - image: gcc:8
         environment:
           CXX: g++
-  haswell-gcc8:
-    docker:
-      - image: gcc:8
-        environment:
-          CXX: g++
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=ON  -DSIMDJSON_IMPLEMENTATION_WESTMERE=OFF -DSIMDJSON_IMPLEMENTATION_FALLBACK=OFF -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
 
   gcc9:
     docker:
       - image: gcc:9
         environment:
           CXX: g++
-  haswell-gcc9:
-    docker:
-      - image: gcc:9
-        environment:
-          CXX: g++
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=ON  -DSIMDJSON_IMPLEMENTATION_WESTMERE=OFF -DSIMDJSON_IMPLEMENTATION_FALLBACK=OFF -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
 
   clang6:
     docker:
       - image: ubuntu:18.04
         environment:
           CXX: clang++-6.0
-  haswell-clang6:
-    docker:
-      - image: ubuntu:18.04
-        environment:
-          CXX: clang++-6.0
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=ON  -DSIMDJSON_IMPLEMENTATION_WESTMERE=OFF -DSIMDJSON_IMPLEMENTATION_FALLBACK=OFF -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
-  westmere-clang6:
-    docker:
-      - image: ubuntu:18.04
-        environment:
-          CXX: clang++-6.0
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=OFF -DSIMDJSON_IMPLEMENTATION_WESTMERE=ON  -DSIMDJSON_IMPLEMENTATION_FALLBACK=OFF -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
-  fallback-clang6:
-    docker:
-      - image: ubuntu:18.04
-        environment:
-          CXX: clang++-6.0
-          CMAKE_IMPLEMENTATION_FLAGS: -DSIMDJSON_IMPLEMENTATION_HASWELL=OFF -DSIMDJSON_IMPLEMENTATION_WESTMERE=OFF -DSIMDJSON_IMPLEMENTATION_FALLBACK=ON  -DSIMDJSON_IMPLEMENTATION_ARM64=OFF
                 
 # Reusable test commands (and initializer for clang 6)
 commands:
@@ -89,45 +41,113 @@ commands:
       - run: make amalgamate
       - run: make test
       - run: make checkperf
-  
+
-  cmake_simple_test: # this version just builds and test
+  cmake_prep:
     steps:
       - run: apt-get update -qq
       - run: apt-get install -y cmake
       - checkout
-      - run: cmake $CMAKE_FLAGS $CMAKE_IMPLEMENTATION_FLAGS
+
+  cmake_build:
+    steps:
+      - cmake_prep
+      - run: cmake $CMAKE_FLAGS $CMAKE_IMPLEMENTATION_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination .
       - run: make all
       - run: tools/json2json -h # Print out the implementation we're using on this hardware
-      - run: ctest --output-on-failure
 
-  cmake_test: # this version builds, install, test and then verify from the installation
+  cmake_test:
     steps:
-      - run: apt-get update -qq
+      - cmake_build
-      - run: apt-get install -y cmake
+      - run: ctest --output-on-failure -L acceptance
-      - checkout
+      - run: ctest --output-on-failure -LE acceptance
-      - run: cmake $CMAKE_FLAGS $CMAKE_IMPLEMENTATION_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination
+
-      - run: make all install
+  cmake_test_all:
-      - run: tools/json2json -h # Print out the implementation we're using on this hardware
+    steps:
-      - run: ctest --output-on-failure
+      - cmake_build
-      - run: echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json # we not only want cmake to build and run tests, but we want also a succesful installation from which we can build, link and run programs
+      - run: ctest --output-on-failure -L acceptance -LE per_implementation
+      - run: SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest --output-on-failure -L per_implementation
+      - run: SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest --output-on-failure -L per_implementation
+      - run: SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest --output-on-failure -L per_implementation
+      - run: ctest --output-on-failure -LE "acceptance|per_implementation" # Everything we haven't run yet, run now.
+
+  # we not only want cmake to build and run tests, but we want also a successful installation from which we can build, link and run programs
+  cmake_install_test: # this version builds, install, test and then verify from the installation
+    steps:
+      - run: make install
+      - run: echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json
 
 jobs:
 
+  # static
+  gcc7:
+    description: Build and run tests on GCC 7 and AVX 2 with a cmake static build
+    executor: gcc7
+    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
+    steps: [ cmake_test_all, cmake_install_test ]
+  clang6:
+    description: Build and run tests on clang 6 and AVX 2 with a cmake static build
+    executor: clang6
+    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
+    steps: [ init_clang6, cmake_test_all, cmake_install_test ]
+  
+  # sanitize
+  sanitize-gcc7:
+    description: Build and run tests on GCC 7 and AVX 2 with a cmake sanitize build
+    executor: gcc7
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
+    steps: [ cmake_test_all ]
+  sanitize-clang6:
+    description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build
+    executor: clang6
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
+    steps: [ init_clang6, cmake_test_all ]
+  
+  # dynamic
+  dynamic-gcc7:
+    description: Build and run tests on GCC 7 and AVX 2 with a cmake dynamic build
+    executor: gcc7
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
+    steps: [ cmake_test, cmake_install_test ]
+  dynamic-clang6:
+    description: Build and run tests on clang 6 and AVX 2 with a cmake dynamic build
+    executor: clang6
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
+    steps: [ init_clang6, cmake_test, cmake_install_test ]
+  
+  # unthreaded
+  unthreaded-gcc7:
+    description: Build and run tests on GCC 7 and AVX 2 *without* threads
+    executor: gcc7
+    environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
+    steps: [ cmake_test, cmake_install_test ]
+  unthreaded-clang6:
+    description: Build and run tests on Clang 6 and AVX 2 *without* threads
+    executor: clang6
+    environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
+    steps: [ init_clang6, cmake_test, cmake_install_test ]
+
+  # noexcept
+  noexcept-gcc7:
+    description: Build and run tests on GCC 7 and AVX 2 with exceptions off
+    executor: gcc7
+    environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
+    steps: [ cmake_test, cmake_install_test ]
+  noexcept-clang6:
+    description: Build and run tests on GCC 7 and AVX 2 with exceptions off
+    executor: clang6
+    environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
+    steps: [ init_clang6, cmake_test, cmake_install_test ]
+
   #
-  # Generic
+  # Misc.
   #
 
   # make (test and checkperf)
   gcc9:
-    description: Build, run tests and check performance on GCC 7
+    description: Build, run tests and check performance on GCC 9
     executor: gcc9
     environment: { EXTRAFLAGS: -Werror }
     steps: [ make_test ]
-  clang6:
-    description: Build, run tests and check performance on clang 6 and AVX 2
-    executor: clang6
-    environment: { EXTRAFLAGS: -Werror }
-    steps: [ init_clang6, make_test ]
   arch-haswell-gcc7:
     description: Build, run tests and check performance on GCC 7 with -march=haswell
     executor: gcc7
@@ -143,167 +163,28 @@ jobs:
     executor: gcc7
     environment: { EXTRAFLAGS: -DSIMDJSON_NO_COMPUTED_GOTO=true }
     steps: [ make_test ]
-  noexcept-gcc7:
-    description: Build and run tests on GCC 7 and AVX 2 with exceptions off
-    executor: gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
-    steps: [ cmake_test ]
-  noexcept-clang6:
-    description: Build and run tests on GCC 7 and AVX 2 with exceptions off
-    executor: clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
-    steps: [ init_clang6, cmake_test ]
-    
-  #
-  # Haswell
-  #
-
-  # static
-  haswell-gcc7:
-    description: Build and run tests on GCC 7 and AVX 2 with a cmake static build
-    executor: haswell-gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
-    steps: [ cmake_test ]
-  haswell-clang6:
-    description: Build and run tests on clang 6 and AVX 2 with a cmake static build
-    executor: haswell-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
-    steps: [ init_clang6, cmake_test ]
-  haswell-gcc8:
-    description: Build and run tests on GCC 8 and AVX 2
-    executor: haswell-gcc8
-    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
-    steps: [ cmake_test ]
-  haswell-gcc9:
-    description: Build and run tests on GCC 9 and AVX 2
-    executor: haswell-gcc9
-    environment: { EXTRAFLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
-    steps: [ cmake_test ]
-  
-  # sanitize
-  haswell-sanitize-gcc7:
-    description: Build and run tests on GCC 7 and AVX 2 with a cmake sanitize build
-    executor: haswell-gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
-    steps: [ cmake_simple_test ]
-  haswell-sanitize-clang6:
-    description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build
-    executor: haswell-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
-    steps: [ init_clang6, cmake_simple_test ]
-  
-  # dynamic
-  haswell-dynamic-gcc7:
-    description: Build and run tests on GCC 7 and AVX 2 with a cmake dynamic build
-    executor: haswell-gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
-    steps: [ cmake_test ]
-  haswell-dynamic-clang6:
-    description: Build and run tests on clang 6 and AVX 2 with a cmake dynamic build
-    executor: haswell-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
-    steps: [ init_clang6, cmake_test ]
-  
-  # unthreaded
-  haswell-unthreaded-gcc7:
-    description: Build and run tests on GCC 7 and AVX 2 *without* threads
-    executor: haswell-gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
-    steps: [ cmake_test ]
-  haswell-unthreaded-clang6:
-    description: Build and run tests on Clang 6 and AVX 2 *without* threads
-    executor: haswell-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
-    steps: [ init_clang6, cmake_test ]
-
-  #
-  # Westmere
-  #
-
-  # static
-  westmere-gcc7:
-    description: Build and run tests on GCC 7 and SSE 4.2 with a cmake static build
-    executor: westmere-gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
-    steps: [ cmake_test ]
-  westmere-clang6:
-    description: Build and run tests on Clang 6 and SSE 4.2 with a cmake static build
-    executor: westmere-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
-    steps: [ init_clang6, cmake_test ]
-
-  # sanitize
-  westmere-sanitize-gcc7:
-    description: Build and run tests on GCC 7 and SSE 4.2 with a cmake sanitize build
-    executor: westmere-gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
-    steps: [ cmake_simple_test ]
-  westmere-sanitize-clang6:
-    description: Build and run tests on Clang 6 and SSE 4.2 with a cmake sanitize build
-    executor: westmere-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
-    steps: [ init_clang6, cmake_simple_test ]
-      
-  #
-  # Fallback
-  #
-
-  # static
-  fallback-gcc7:
-    description: Build and run tests on GCC 7 and fallback implementation with a cmake static build
-    executor: fallback-gcc7
-    steps: [ cmake_test ]
-  fallback-clang6:
-    description: Build and run tests on Clang 6 and fallback implementation with a cmake static build
-    executor: fallback-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
-    steps: [ init_clang6, cmake_test ]
-
-  # sanitize
-  fallback-sanitize-gcc7:
-    description: Build and run tests on GCC 7 and fallback implementation with a cmake sanitize build
-    executor: fallback-gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
-    steps: [ cmake_simple_test ]
-  fallback-sanitize-clang6:
-    description: Build and run tests on Clang 6 and fallback implementation with a cmake sanitize build
-    executor: fallback-clang6
-    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON }
-    steps: [ init_clang6, cmake_simple_test ]
 
 workflows:
   version: 2.1
   build_and_test:
     jobs:
-      - gcc9
+      # full multi-implementation tests
-      - haswell-gcc7
+      - gcc7
-      - westmere-gcc7
-      - fallback-gcc7
-
       - clang6
-      - haswell-clang6
-      - westmere-clang6
-      - fallback-clang6
 
-      - haswell-dynamic-gcc7
+      # full single-implementation tests
-      - haswell-dynamic-clang6
+      - sanitize-gcc7
-      - haswell-unthreaded-gcc7
+      - sanitize-clang6
-      - haswell-unthreaded-clang6
+      - dynamic-gcc7
+      - dynamic-clang6
+      - unthreaded-gcc7
+      - unthreaded-clang6
 
-      - haswell-sanitize-gcc7
+      # quicker make single-implementation tests
-      - haswell-sanitize-clang6
+      - gcc9
-      - westmere-sanitize-gcc7
-      - westmere-sanitize-clang6
-      - fallback-sanitize-gcc7
-      - fallback-sanitize-clang6
-
-      - noexcept-gcc7
-      - noexcept-clang6
-      - haswell-gcc8
-      - haswell-gcc9
       - arch-haswell-gcc7
       - arch-nehalem-gcc7
       - no-computed-goto-gcc7
-
+    
       # TODO add windows: https://circleci.com/docs/2.0/configuration-reference/#windows
 

examples/quickstart/CMakeLists.txt

@@ -45,12 +45,10 @@ if (NOT MSVC)
     add_quickstart_test(quickstart quickstart.cpp)
     add_quickstart_test(quickstart11 quickstart.cpp c++11)
     add_quickstart_test(quickstart14 quickstart.cpp c++14)
-    set_property( TEST quickstart quickstart11 APPEND PROPERTY LABELS quicktests )
+    set_property( TEST quickstart quickstart11 APPEND PROPERTY LABELS acceptance compiletests )
-    set_property( TEST quickstart14            APPEND PROPERTY LABELS slowtests  )
   endif()
 
   add_quickstart_test(quickstart_noexceptions quickstart_noexceptions.cpp "" true)
   add_quickstart_test(quickstart_noexceptions11 quickstart_noexceptions.cpp c++11 true)
-  set_property( TEST quickstart_noexceptions APPEND PROPERTY LABELS quicktests )
+  set_property( TEST quickstart_noexceptions APPEND PROPERTY LABELS acceptance compile )
-  set_property( TEST quickstart_noexceptions11 APPEND PROPERTY LABELS slowtests )
 endif()

tests/CMakeLists.txt

@@ -1,55 +1,63 @@
 # Helper so we don't have to repeat ourselves so much
-function(add_cpp_test TEST_NAME TEST_FILE)
+# Usage: add_cpp_test(testname [COMPILE_ONLY] [SOURCES a.cpp b.cpp ...] [LABELS acceptance per_implementation ...])
-  # If a source file is passed, add an executable
+# SOURCES defaults to testname.cpp if not specified.
-  add_executable(${TEST_NAME} ${TEST_FILE})
+function(add_cpp_test TEST_NAME)
-  add_test(${TEST_NAME} ${TEST_NAME})
+  # Parse arguments
-  if ($ARGN)
+  cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;WILL_FAIL" "" "SOURCES;LABELS")
-    set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${ARGN})
+  if (NOT ARGS_SOURCES)
+    list(APPEND ARGS_SOURCES ${TEST_NAME}.cpp)
+  endif()
+  if (COMPILE_ONLY)
+    list(APPEND ${ARGS_LABELS} compile)
+  endif()
+
+  # Add executable
+  add_executable(${TEST_NAME} ${ARGS_SOURCES})
+
+  # Add test
+  if (ARGS_COMPILE_ONLY)
+    add_test(
+      NAME ${TEST_NAME}
+      COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
+      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    )
+    set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
   else()
-    set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS slowtests)
+    add_test(${TEST_NAME} ${TEST_NAME})
+  endif()
+
+  if (ARGS_LABELS)
+    set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${ARGS_LABELS})
+  endif()
+
+  if (ARGS_WILL_FAIL)
+    set_property(TEST ${TEST_NAME} PROPERTY WILL_FAIL TRUE)
   endif()
 endfunction()
-
-function(add_compile_test TEST_NAME TEST_FILE)
-  add_executable(${TEST_NAME} ${TEST_FILE})
-  set_target_properties(${TEST_NAME} PROPERTIES
-                        EXCLUDE_FROM_ALL TRUE
-                        EXCLUDE_FROM_DEFAULT_BUILD TRUE)
-  add_test(
-    NAME ${TEST_NAME}
-    COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-  )
-  if (${ARGN}) # Labels
-    set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${ARGN})
-  else()
-    set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS slowtests)
-  endif()
-endfunction(add_compile_test)
-
 # Most tests need test data, and many need windows headers.
 link_libraries(simdjson-flags test-data simdjson-windows-headers)
 
+include(${PROJECT_SOURCE_DIR}/tests/add_cpp_test.cmake)
+
 #
 # These tests explicitly do #include "simdjson.cpp" so they can override stuff
 #
 if (NOT MSVC) # Can't get simdjson-source to compile on Windows for some reason.
-  add_cpp_test(numberparsingcheck numberparsingcheck.cpp quicktests)
+  add_cpp_test(numberparsingcheck LABELS acceptance per_implementation)
   target_link_libraries(numberparsingcheck simdjson-include-source)
-  add_cpp_test(stringparsingcheck stringparsingcheck.cpp quicktests)
+  add_cpp_test(stringparsingcheck LABELS acceptance per_implementation)
   target_link_libraries(stringparsingcheck simdjson-include-source)
 endif()
 
 # All remaining tests link with simdjson proper
 link_libraries(simdjson)
-add_cpp_test(basictests basictests.cpp quicktests)
+add_cpp_test(basictests LABELS acceptance per_implementation)
-add_cpp_test(checkimplementation checkimplementation.cpp quicktests)
+add_cpp_test(errortests LABELS acceptance per_implementation)
-add_cpp_test(errortests errortests.cpp quicktests)
+add_cpp_test(integer_tests LABELS acceptance per_implementation)
-add_cpp_test(integer_tests integer_tests.cpp quicktests)
+add_cpp_test(jsoncheck LABELS acceptance per_implementation)
-add_cpp_test(jsoncheck jsoncheck.cpp quicktests)
+add_cpp_test(parse_many_test LABELS acceptance per_implementation)
-add_cpp_test(parse_many_test parse_many_test.cpp quicktests)
+add_cpp_test(pointercheck LABELS acceptance per_implementation)
-add_cpp_test(pointercheck pointercheck.cpp quicktests)
+add_cpp_test(extracting_values_example LABELS acceptance per_implementation)
-add_cpp_test(extracting_values_example extracting_values_example.cpp quicktests)
 
 # Script tests
 if (NOT MSVC) # Can't run .sh on windows
@@ -62,7 +70,7 @@ if (NOT MSVC) # Can't run .sh on windows
     WORKING_DIRECTORY $<TARGET_FILE_DIR:minify>
   )
   set_property(TEST testjson2json APPEND PROPERTY DEPENDS minify json2json)
-  set_property(TEST testjson2json APPEND PROPERTY LABELS slowtests)
+  set_property(TEST testjson2json APPEND PROPERTY LABELS per_implementation)
 
   #
   # Competition parse test
@@ -73,27 +81,32 @@ if (NOT MSVC) # Can't run .sh on windows
 
     add_test(issue150 ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
     set_property(TEST issue150 APPEND PROPERTY DEPENDS allparserscheckfile)
-    set_property(TEST issue150 APPEND PROPERTY LABELS slowtests)
+    set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation)
   endif()
 endif()
 
 if (NOT MSVC)
   #
-  # json2json tool tests
+  # json2json tool test: check that json2json can parse twitter.json
   #
 
+  # This tests validates that the implementation is what we think it is if we get passed
+  # SIMDJSON_FORCE_IMPLEMENTATION, so we know we're testing what we think we're testing
+  add_cpp_test(checkimplementation LABELS per_implementation)
+
   add_test(NAME json2json COMMAND $<TARGET_FILE:json2json> ${EXAMPLE_JSON})
+  set_property(TEST issue150 APPEND PROPERTY LABELS acceptance per_implementation)
 
   #
-  # SIMDJSON_FORCE_IMPLEMENTATION tests
+  # SIMDJSON_FORCE_IMPLEMENTATION tests: run json2json with SIMDJSON
   #
   if (SIMDJSON_IMPLEMENTATION_FALLBACK)
     add_test(
-      NAME simdjson_force_implementation_fallback
+      NAME simdjson_force_implementation
       COMMAND
         ${CMAKE_COMMAND} -E env
         SIMDJSON_FORCE_IMPLEMENTATION=fallback
-        $<TARGET_FILE:json2json> ${EXAMPLE_JSON}
+        $<TARGET_FILE:checkimplementation>
     )
   endif()
   add_test(
@@ -113,35 +126,19 @@ endif()
 # Don't add the tests if we're on VS2017 or older; they don't succeed.
 if(NOT (MSVC AND MSVC_VERSION LESS 1920))
   if(SIMDJSON_EXCEPTIONS)
-    add_compile_test(readme_examples readme_examples.cpp quicktests)
+    add_cpp_test(readme_examples COMPILE_ONLY LABELS acceptance)
-    set_property(
+    add_cpp_test(readme_examples11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
-      TEST readme_examples
-      APPEND PROPERTY LABELS quicktests
-    )
-
-    add_compile_test(readme_examples11 readme_examples.cpp quicktests)
     set_target_properties(readme_examples11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF)
-    if (!MSVC)
-      target_compile_options(readme_examples11 PRIVATE -Werror)
-    endif()
   endif()
 
-  add_compile_test(readme_examples_noexceptions readme_examples_noexceptions.cpp quicktests)
+  add_cpp_test(readme_examples_noexceptions COMPILE_ONLY LABELS acceptance)
 
-  add_compile_test(readme_examples_noexceptions11 readme_examples_noexceptions.cpp quicktests)
+  add_cpp_test(readme_examples_noexceptions11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples_noexceptions.cpp)
   set_target_properties(readme_examples_noexceptions11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF)
-  if (!MSVC)
-    target_compile_options(readme_examples_noexceptions11 PRIVATE -Werror)
-  endif()
 
   # Compile tests that *should fail*
-  add_compile_test(readme_examples_will_fail_with_exceptions_off readme_examples.cpp quicktests)
+  add_cpp_test(readme_examples_will_fail_with_exceptions_off WILL_FAIL COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
   target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0)
-  set_tests_properties(readme_examples_will_fail_with_exceptions_off PROPERTIES WILL_FAIL TRUE)
-  set_property(
-    TEST readme_examples_noexceptions readme_examples_will_fail_with_exceptions_off
-    APPEND PROPERTY LABELS quicktests
-  )
 
 endif()
 

tests/add_cpp_test.cmake

@@ -0,0 +1,36 @@
+# Helper so we don't have to repeat ourselves so much
+# Usage: add_cpp_test(testname [COMPILE_ONLY] [SOURCES a.cpp b.cpp ...] [LABELS acceptance per_implementation ...])
+# SOURCES defaults to testname.cpp if not specified.
+function(add_cpp_test TEST_NAME)
+  # Parse arguments
+  cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;WILL_FAIL" "" "SOURCES;LABELS")
+  if (NOT ARGS_SOURCES)
+    list(APPEND ARGS_SOURCES ${TEST_NAME}.cpp)
+  endif()
+  if (COMPILE_ONLY)
+    list(APPEND ${ARGS_LABELS} compile)
+  endif()
+
+  # Add executable
+  add_executable(${TEST_NAME} ${ARGS_SOURCES})
+
+  # Add test
+  if (ARGS_COMPILE_ONLY)
+    add_test(
+      NAME ${TEST_NAME}
+      COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
+      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+    )
+    set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
+  else()
+    add_test(${TEST_NAME} ${TEST_NAME})
+  endif()
+
+  if (ARGS_LABELS)
+    set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${ARGS_LABELS})
+  endif()
+
+  if (ARGS_WILL_FAIL)
+    set_property(TEST ${TEST_NAME} PROPERTY WILL_FAIL TRUE)
+  endif()
+endfunction()

tests/compilation_failure_tests/CMakeLists.txt

@@ -5,36 +5,21 @@
 # the macro COMPILATION_TEST_USE_FAILING_CODE set to 0 or 1.
 #
 
-# internal function for add_dual_compile_test
-# which adds a target and a test target trying to compile it
-function(detail_add_dual_compile_test TEST_NAME TEST_FILE)
-  add_executable(${TEST_NAME} ${TEST_FILE})
-  set_target_properties(${TEST_NAME} PROPERTIES
-                        EXCLUDE_FROM_ALL TRUE
-                        EXCLUDE_FROM_DEFAULT_BUILD TRUE)
-  add_test(NAME ${TEST_NAME}
-           COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
-           WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-endfunction(detail_add_dual_compile_test)
-
 # adds a compilation test. two targets are created, one expected to
 # succeed compilation and one that is expected to fail.
-function(add_dual_compile_test TEST_NAME TEST_FILE)
+function(add_dual_compile_test TEST_NAME)
-    detail_add_dual_compile_test(${TEST_NAME}_should_compile ${TEST_FILE})
+  add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY)
-    detail_add_dual_compile_test(${TEST_NAME}_should_not_compile ${TEST_FILE})
+  add_cpp_test(${TEST_NAME}_should_not_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY WILL_FAIL LABELS acceptance)
-    target_compile_definitions(${TEST_NAME}_should_compile     PRIVATE COMPILATION_TEST_USE_FAILING_CODE=0)
+  target_compile_definitions(${TEST_NAME}_should_not_compile PRIVATE COMPILATION_TEST_USE_FAILING_CODE=1)
-    target_compile_definitions(${TEST_NAME}_should_not_compile PRIVATE COMPILATION_TEST_USE_FAILING_CODE=1)
-    set_tests_properties(${TEST_NAME}_should_compile           PROPERTIES WILL_FAIL FALSE)
-    set_tests_properties(${TEST_NAME}_should_not_compile       PROPERTIES WILL_FAIL TRUE)
 endfunction(add_dual_compile_test)
 
 
-add_dual_compile_test(example_compiletest               example_compiletest.cpp)
+add_dual_compile_test(example_compiletest)
 # These don't compile with exceptions off
 if (SIMDJSON_EXCEPTIONS)
-  add_dual_compile_test(dangling_parser_load              dangling_parser_load.cpp)
+  add_dual_compile_test(dangling_parser_load)
-  add_dual_compile_test(dangling_parser_parse_uint8       dangling_parser_parse_uint8.cpp)
+  add_dual_compile_test(dangling_parser_parse_uint8)
-  add_dual_compile_test(dangling_parser_parse_uchar       dangling_parser_parse_uchar.cpp)
+  add_dual_compile_test(dangling_parser_parse_uchar)
-  add_dual_compile_test(dangling_parser_parse_stdstring   dangling_parser_parse_stdstring.cpp)
+  add_dual_compile_test(dangling_parser_parse_stdstring)
-  add_dual_compile_test(dangling_parser_parse_padstring   dangling_parser_parse_padstring.cpp)
+  add_dual_compile_test(dangling_parser_parse_padstring)
 endif()