jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Various minor tweaks.

This commit is contained in:
Daniel Lemire 2019-07-04 17:19:05 -04:00
parent 36471c23ce
commit 2b2d93b05f
10 changed files with 74 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.appveyor.yml
View File

@ -9,9 +9,14 @@ clone_folder: c:\projects\simdjson
platform:
- x64
environment:
matrix:
- AVXFLAG: "OFF"
- AVXFLAG: "ON"
build_script:
- mkdir build
- cd build
- ps: cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..
- ps: cmake -DSIMDJSON_DISABLE_AVX="$env:AVXFLAG" -DCMAKE_GENERATOR_PLATFORM=x64 ..
- cmake --build .
- ctest --verbose

3
.travis.yml
View File

@ -19,3 +19,6 @@ script:
- make test
- make clean
- make SANITIZEGOLD=1 test
- make clean
- ARCHFLAGS="-march=nehalem" make
- ARCHFLAGS="-march=nehalem" make test

2
CMakeLists.txt
View File

@ -5,6 +5,8 @@ if(ltoresult)
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
endif()
option(SIMDJSON_DISABLE_AVX "Forcefully disable AVX even if hardware supports it" OFF)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_MACOSX_RPATH OFF)

17
Makefile
View File

@ -9,16 +9,21 @@ COREDEPSINCLUDE = -Idependencies/rapidjson/include -Idependencies/sajson/include
EXTRADEPSINCLUDE = -Idependencies/jsoncppdist -Idependencies/json11 -Idependencies/fastjson/src -Idependencies/fastjson/include -Idependencies/gason/src -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src
# users can provide their own additional flags with make EXTRAFLAGS=something
architecture:=$(shell arch)
CXXFLAGS = -std=c++17 -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux $(EXTRAFLAGS)
CFLAGS = -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS)
####
# If you want to specify your own target architecture,
# then define ARCHFLAGS. Otherwise, we set good default.
###
ifeq ($(architecture),aarch64)
CXXFLAGS += -march=armv8-a+crc+crypto
CFLAGS += -march=armv8-a+crc+crypto
ARCHFLAGS ?= -march=armv8-a+crc+crypto
else
CXXFLAGS += -march=native
CFLAGS += -march=native
ARCHFLAGS ?= -march=native
endif
CXXFLAGS = $(ARCHFLAGS) -std=c++17 -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux $(EXTRAFLAGS)
CFLAGS = $(ARCHFLAGS) -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS)
# This is a convenience flag
ifdef SANITIZEGOLD
SANITIZE = 1

11
README.md
View File

@ -51,7 +51,7 @@ On a Skylake processor, the parsing speeds (in GB/s) of various processors on th
## Requirements
- We support platforms like Linux or macOS, as well as Windows through Visual Studio 2017 or later.
- A processor with AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013 and AMD processors starting with the Zen microarchitecture released 2017).
- A processor with AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013 and AMD processors starting with the Zen microarchitecture released 2017) or at least SSE 4.2 (i.e., Intel processors going back to Nehalem released in 2008 or AMD processors starting with the Jaguar used in the PS4 and XBox One).
- A recent C++ compiler (e.g., GNU GCC or LLVM CLANG or Visual Studio 2017), we assume C++17. GNU GCC 7 or better or LLVM's clang 6 or better.
- Some benchmark scripts assume bash and other common utilities, but they are optional.
@ -168,7 +168,7 @@ int main(int argc, char *argv[]) {
}
```
We require hardware support for AVX2 instructions. You have to make sure that you instruct your
On Intel and AMD processors, we get best performance by using the hardware support for AVX2 instructions. You have to make sure that you instruct your
compiler to use these instructions as needed. Under compilers such as GNU GCC or LLVM clang, the
flag `-march=native` used on a recent Intel processor (Haswell or better) is sufficient. For portability
of the binary files you can also specify directly the Haswell processor (`-march=haswell`). You may
@ -260,14 +260,15 @@ make test
## Usage (CMake on Windows using Visual Studio)
We assume you have a common Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later).
We assume you have a common Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later) or SSE 4.2 (2008 Nehalem or later).
- Grab the simdjson code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/).
- Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that `cmake` be made available from the command line. Please choose a recent version of cmake.
- Create a subdirectory within simdjson, such as `VisualStudio`.
- Using a shell, go to this newly created directory.
- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `VisualStudio` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.)
- This last command created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`.
- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `VisualStudio` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.) This will build the code with AVX2 instructions. If your target processor does not support AVX2, you need to replace `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` by `cmake -DSIMDJSON_DISABLE_AVX=on -DCMAKE_GENERATOR_PLATFORM=x64 ..` . That is, you need to set the flag to forcefully disable AVX support since we compile with AVX2 instructions *by default*.
- This last command (`cmake ...`) created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`.
## Usage (Using `vcpkg` on Windows, Linux and MacOS)

2
include/simdjson/jsonminifier.h
View File

@ -5,9 +5,11 @@
#include <cstdint>
namespace simdjson {
// Take input from buf and remove useless whitespace, write it to out; buf and
// out can be the same pointer. Result is null terminated,
// return the string length (minus the null termination).
// The accelerated version of this function only runs on AVX2 hardware.
size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out);

2
include/simdjson/simdutf8check.h
View File

@ -322,8 +322,6 @@ avxcheckUTF8Bytes(__m256i current_bytes,
return pb;
}
#else // __AVX2__
#warning "We require AVX2 support!"
#endif // __AVX2__
}
#endif

43
include/simdjson/stage1_find_marks.h
View File

@ -126,6 +126,19 @@ uint64_t neonmovemask_bulk(uint8x16_t p0, uint8x16_t p1, uint8x16_t p2, uint8x16
template<instruction_set T>
uint64_t compute_quote_mask(uint64_t quote_bits);
namespace {
// for when clmul is unavailable
[[maybe_unused]] uint64_t portable_compute_quote_mask(uint64_t quote_bits) {
uint64_t quote_mask = quote_bits ^ (quote_bits << 1);
quote_mask = quote_mask ^ (quote_mask << 2);
quote_mask = quote_mask ^ (quote_mask << 4);
quote_mask = quote_mask ^ (quote_mask << 8);
quote_mask = quote_mask ^ (quote_mask << 16);
quote_mask = quote_mask ^ (quote_mask << 32);
return quote_mask;
}
}
// In practice, if you have NEON or __PCLMUL__, you would
// always want to use them, but it might be useful, for research
// purposes, to disable it willingly, that's what SIMDJSON_AVOID_CLMUL
@ -135,13 +148,7 @@ uint64_t compute_quote_mask(uint64_t quote_bits);
#ifdef SIMDJSON_AVOID_CLMUL
template<instruction_set T> really_inline
uint64_t compute_quote_mask(uint64_t quote_bits) {
uint64_t quote_mask = quote_bits ^ (quote_bits << 1);
quote_mask = quote_mask ^ (quote_mask << 2);
quote_mask = quote_mask ^ (quote_mask << 4);
quote_mask = quote_mask ^ (quote_mask << 8);
quote_mask = quote_mask ^ (quote_mask << 16);
quote_mask = quote_mask ^ (quote_mask << 32);
return quote_mask;
return portable_compute_quote_mask(quote_bits);
}
#else
template<instruction_set>
@ -150,6 +157,8 @@ uint64_t compute_quote_mask(uint64_t quote_bits);
#ifdef __AVX2__
template<> really_inline
uint64_t compute_quote_mask<instruction_set::avx2>(uint64_t quote_bits) {
// There should be no such thing with a processing supporting avx2
// but not clmul.
uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
_mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0));
return quote_mask;
@ -159,23 +168,25 @@ uint64_t compute_quote_mask<instruction_set::avx2>(uint64_t quote_bits) {
#ifdef __SSE4_2__
template<> really_inline
uint64_t compute_quote_mask<instruction_set::sse4_2>(uint64_t quote_bits) {
uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
// CLMUL is supported on some SSE42 hardware such as Sandy Bridge,
// but not on others.
#ifdef __PCLMUL__
return _mm_cvtsi128_si64(_mm_clmulepi64_si128(
_mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0));
return quote_mask;
#else
return portable_compute_quote_mask(quote_bits);
#endif
}
#endif
#ifdef __ARM_NEON
template<> really_inline
uint64_t compute_quote_mask<instruction_set::neon>(uint64_t quote_bits) {
#ifdef __PCLMUL__ // Might cause problems on runtime dispatch
uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
_mm_set_epi64x(0ULL, quote_bits),
_mm_set1_epi8(0xFF), 0));
#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension
return vmull_p64( -1ULL, quote_bits);
#else
uint64_t quote_mask = vmull_p64( -1ULL, quote_bits);
#endif
return quote_mask;
return portable_compute_quote_mask(quote_bits);
#endif
}
#endif
#endif // SIMDJSON_AVOID_CLMUL

9
tests/CMakeLists.txt
View File

@ -7,7 +7,8 @@ endif()
add_cpp_test(basictests)
add_cpp_test(jsoncheck)
add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp)
target_compile_definitions(singleheader PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json")
target_link_libraries(singleheader ${SIMDJSON_LIB_NAME})
add_test(singleheader singleheader)
## This causes problems
# add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp)
# target_compile_definitions(singleheader PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json")
# target_link_libraries(singleheader ${SIMDJSON_LIB_NAME})
# add_test(singleheader singleheader)

18
tools/cmake/FindOptions.cmake
View File

@ -13,15 +13,21 @@ if(SIMDJSON_SANITIZE)
endif()
# some compilers like clang do not automagically define __AVX2__ and __BMI2__ even when the hardware supports it
if(NOT MSVC)
if(SIMDJSON_DISABLE_AVX)
if(NOT MSVC)
set (OPT_FLAGS "${OPT_FLAGS} -mno-avx -mno-bmi -mno-pclmul -msse4.2")
else()
set (OPT_FLAGS "${OPT_FLAGS}")
endif()
else()
# some compilers like clang do not automagically define __AVX2__ and __BMI2__ even when the hardware supports it
if(NOT MSVC)
set (OPT_FLAGS "${OPT_FLAGS} -mavx2 -mbmi -mbmi2 -mpclmul")
else()
set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2 /std:c++latest")
else()
set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2")
endif()
endif()
if(NOT MSVC)
set(CXXSTD_FLAGS "-std=c++17 -fPIC")
endif()