jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Merge branch 'master' of github.com:lemire/simdjson

This commit is contained in:
Daniel Lemire 2019-08-12 16:05:31 -04:00
parent 5c538dd9d6 875e2f9d0d
commit 7f27e1e0e1
17 changed files with 486 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.appveyor.yml
View File

@ -19,4 +19,4 @@ build_script:
- cd build
- ps: cmake -DSIMDJSON_BUILD_STATIC="$env:SIMDJSON_BUILD_STATIC" -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=x64 ..
- cmake --build .
- ctest --verbose
- ctest --verbose --output-on-failure

138
.circleci/config.yml
View File

@ -25,19 +25,46 @@ jobs:
command: make quiettest amalgamate
- run:
name: Building (gcc, cmake)
name: Building (gcc, cmake, dynamic)
command: |
mkdir build
cd build
cmake ..
cmake -DSIMDJSON_BUILD_STATIC=OFF ..
make
- run:
name: Running tests (gcc, cmake)
name: Running tests (gcc, cmake, dynamic)
command: |
cd build
make test
- run:
name: Building (gcc, cmake, static)
command: |
mkdir buildstatic
cd buildstatic
cmake -DSIMDJSON_BUILD_STATIC=ON ..
make
- run:
name: Running tests (gcc, cmake, static)
command: |
cd buildstatic
make test
- run:
name: Building (gcc, cmake, sanitize)
command: |
mkdir buildsani
cd buildsani
cmake -DSIMDJSON_SANITIZE=ON -DSIMDJSON_BUILD_STATIC=OFF ..
make
- run:
name: Running tests (gcc, cmake, sanitize)
command: |
cd buildsani
make test
"gccnoavx":
docker:
- image: ubuntu:18.04
@ -63,19 +90,46 @@ jobs:
command: ARCHFLAGS="-march=nehalem" make quiettest amalgamate
- run:
name: Building (gcc, cmake)
name: Building (gcc, cmake, dynamic)
command: |
mkdir build
cd build
cmake -DSIMDJSON_DISABLE_AVX=on ..
cmake -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=OFF ..
make
- run:
name: Running tests (gcc, cmake)
name: Running tests (gcc, cmake, dynamic)
command: |
cd build
make test
- run:
name: Building (gcc, cmake, static)
command: |
mkdir buildstatic
cd buildstatic
cmake -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_BUILD_STATIC=ON ..
make
- run:
name: Running tests (gcc, cmake, static)
command: |
cd buildstatic
make test
- run:
name: Building (gcc, cmake, sanitize)
command: |
mkdir buildsani
cd buildsani
cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_SANITIZE=ON ..
make
- run:
name: Running tests (gcc, cmake, sanitize)
command: |
cd buildsani
make test
"clang":
docker:
- image: ubuntu:18.04
@ -101,19 +155,47 @@ jobs:
command: make quiettest amalgamate
- run:
name: Building (clang, cmake)
name: Building (clang, cmake, dynamic)
command: |
mkdir build
cd build
cmake ..
cmake -DSIMDJSON_BUILD_STATIC=OFF ..
make
- run:
name: Running tests (clang, cmake)
name: Running tests (clang, cmake, dynamic)
command: |
cd build
make test
- run:
name: Building (clang, cmake, static)
command: |
mkdir buildstatic
cd buildstatic
cmake -DSIMDJSON_BUILD_STATIC=ON ..
make
- run:
name: Running tests (clang, cmake, static)
command: |
cd buildstatic
make test
- run:
name: Building (clang, cmake, sanitize)
command: |
mkdir buildsani
cd buildsani
cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON ..
make
- run:
name: Running tests (clang, cmake, sanitize)
command: |
cd buildsani
make test
"clangnoavx":
docker:
- image: ubuntu:18.04
@ -139,19 +221,49 @@ jobs:
command: ARCHFLAGS="-march=nehalem" make quiettest amalgamate
- run:
name: Building (clang, cmake)
name: Building (clang, cmake, dynamic)
command: |
mkdir build
cd build
cmake -DSIMDJSON_DISABLE_AVX=on ..
cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_DISABLE_AVX=on ..
make
- run:
name: Running tests (clang, cmake)
name: Running tests (clang, cmake, dynamic)
command: |
cd build
make test
- run:
name: Building (clang, cmake, static)
command: |
mkdir buildstatic
cd buildstatic
cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DISABLE_AVX=on ..
make
- run:
name: Running tests (clang, cmake, static)
command: |
cd buildstatic
make test
- run:
name: Building (clang, cmake, sanitize)
command: |
mkdir buildsani
cd buildsani
cmake -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_DISABLE_AVX=ON -DSIMDJSON_SANITIZE=ON ..
make
- run:
name: Running tests (clang, cmake, sanitize)
command: |
cd buildsani
make test
workflows:
version: 2
build_and_test:
@ -159,4 +271,4 @@ workflows:
- "clang"
- "gcc"
- "clangnoavx"
- "gccnoavx"
- "gccnoavx"

156
.drone.yml
View File

@ -12,6 +12,10 @@ steps:
- make
- make quiettest
- make amalgamate
- name: checkperf
image: gcc:8
commands:
- make checkperf
---
kind: pipeline
name: arm64
@ -27,6 +31,10 @@ steps:
- make
- make quiettest
- make amalgamate
- name: checkperf
image: gcc:8
commands:
- make checkperf
---
kind: pipeline
name: stylecheck
@ -63,7 +71,7 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: amd64_clang_cmake_static
@ -85,7 +93,7 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: amd64_gcc_cmake_static
@ -107,7 +115,7 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: amd64_gcc_cmake_dynamic
@ -129,7 +137,7 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: arm64_clang_cmake_dynamic
@ -151,7 +159,7 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: arm64_gcc_cmake_dynamic
@ -173,7 +181,7 @@ steps:
- mkdir build && cd build
- cmake $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
commands:
- apt-get update -y
@ -182,7 +190,7 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: arm64_clang_cmake_static
@ -204,7 +212,7 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: arm64_gcc_cmake_static
@ -226,9 +234,139 @@ steps:
- mkdir build && cd build
- cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
- make -j
- ctest
- ctest --output-on-failure
---
kind: pipeline
name: arm64_clang_cmake_sanitize
platform:
os: linux
arch: arm64
steps:
- name: Build and Test
image: ubuntu:18.04
environment:
CC: clang
CXX: clang++
commands:
- apt-get update -y
- apt-get install -y make $CC g++ cmake
- $CC --version
- mkdir build && cd build
- cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
- make -j
- ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
---
kind: pipeline
name: arm64_gcc_cmake_sanitize
platform:
os: linux
arch: arm64
steps:
- name: Build and Test
image: ubuntu:18.04
environment:
CC: gcc
CXX: g++
commands:
- apt-get update -y
- apt-get install -y make $CC g++ cmake
- $CC --version
- mkdir build && cd build
- cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
- make -j
- ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
---
kind: pipeline
name: amd64_clang_cmake_sanitize
platform:
os: linux
arch: amd64
steps:
- name: Build and Test
image: ubuntu:18.04
environment:
CC: clang
CXX: clang++
commands:
- apt-get update -y
- apt-get install -y make $CC g++ cmake
- $CC --version
- mkdir build && cd build
- cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
- make -j
- ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
---
kind: pipeline
name: amd64_gcc_cmake_sanitize
platform:
os: linux
arch: amd64
steps:
- name: Build and Test
image: ubuntu:18.04
environment:
CC: gcc
CXX: g++
commands:
- apt-get update -y
- apt-get install -y make $CC g++ cmake
- $CC --version
- mkdir build && cd build
- cmake -DSIMDJSON_SANITIZE=ON $CMAKE_FLAGS ..
- make -j
- ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
---
kind: pipeline
name: amd64_clang_cmake_sanitize_noavx
platform:
os: linux
arch: amd64
steps:
- name: Build and Test
image: ubuntu:18.04
environment:
CC: clang
CXX: clang++
commands:
- apt-get update -y
- apt-get install -y make $CC g++ cmake
- $CC --version
- mkdir build && cd build
- cmake -DSIMDJSON_SANITIZE=ON -DSIMDJSON_DISABLE_AVX=ON $CMAKE_FLAGS ..
- make -j
- ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure
---
kind: pipeline
name: amd64_gcc_cmake_sanitize_noavx
platform:
os: linux
arch: amd64
steps:
- name: Build and Test
image: ubuntu:18.04
environment:
CC: gcc
CXX: g++
commands:
- apt-get update -y
- apt-get install -y make $CC g++ cmake
- $CC --version
- mkdir build && cd build
- cmake -DSIMDJSON_SANITIZE=ON -DSIMDJSON_DISABLE_AVX=ON $CMAKE_FLAGS ..
- make -j
- ASAN_OPTIONS="detect_leaks=0" ctest --output-on-failure

6
Makefile
View File

@ -123,6 +123,12 @@ $(JSON_INCLUDE) $(SAJSON_INCLUDE) $(RAPIDJSON_INCLUDE) $(JSON11_INCLUDE) $(FASTJ
parse: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
$(CXX) $(CXXFLAGS) -o parse $(LIBFILES) benchmark/parse.cpp $(LIBFLAGS)
perfdiff: benchmark/perfdiff.cpp
$(CXX) $(CXXFLAGS) -o perfdiff benchmark/perfdiff.cpp $(LIBFLAGS)
checkperf:
bash ./scripts/checkperf.sh
statisticalmodel: benchmark/statisticalmodel.cpp $(HEADERS) $(LIBFILES)
$(CXX) $(CXXFLAGS) -o statisticalmodel $(LIBFILES) benchmark/statisticalmodel.cpp $(LIBFLAGS)

1
benchmark/CMakeLists.txt
View File

@ -6,3 +6,4 @@ target_include_directories(${SIMDJSON_LIB_NAME}
add_cpp_benchmark(parse)
add_cpp_benchmark(statisticalmodel)
add_executable(perfdiff perfdiff.cpp)

19
benchmark/Dockerfile Normal file
View File

@ -0,0 +1,19 @@
# From the ROOT, run:
# docker build -t simdjsonbench -f benchmark/Dockerfile . && docker run --privileged -t simdjsonbench
FROM gcc:8.3
# # Build latest
# ENV latest_release=v0.2.1
# WORKDIR /usr/src/$latest_release/
# RUN git clone --depth 1 https://github.com/lemire/simdjson/ -b $latest_release .
# RUN make parse
# # Build master
# WORKDIR /usr/src/master/
# RUN git clone --depth 1 https://github.com/lemire/simdjson/ .
# RUN make parse
# Build the current source
COPY . /usr/src/current/
WORKDIR /usr/src/current/
RUN make checkperf

61
benchmark/parse.cpp
View File

@ -124,11 +124,20 @@ int main(int argc, char *argv[]) {
bool json_output = false;
bool force_one_iteration = false;
bool just_data = false;
int32_t iterations = -1;
int32_t warmup_iterations = -1;
#ifndef _MSC_VER
int c;
while ((c = getopt(argc, argv, "1vdt")) != -1) {
while ((c = getopt(argc, argv, "1vdtn:w:")) != -1) {
switch (c) {
case 'n':
iterations = atoi(optarg);
break;
case 'w':
warmup_iterations = atoi(optarg);
break;
case 't':
just_data = true;
break;
@ -174,12 +183,21 @@ int main(int argc, char *argv[]) {
std::cout << "[verbose] loaded " << filename << " (" << p.size()
<< " bytes)" << std::endl;
}
#if defined(DEBUG)
const uint32_t iterations = 1;
#else
const uint32_t iterations =
force_one_iteration ? 1 : (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
#endif
if (iterations == -1) {
#if defined(DEBUG)
iterations = 1;
#else
iterations = force_one_iteration ? 1 : (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
#endif
}
if (warmup_iterations == -1) {
#if defined(DEBUG)
warmup_iterations = 0;
#else
warmup_iterations = (p.size() < 1 * 1000 * 1000) ? 10 : 1;
#endif
}
std::vector<double> res;
res.resize(iterations);
if (!just_data)
@ -214,9 +232,33 @@ int main(int argc, char *argv[]) {
unsigned long cref0 = 0, cref1 = 0, cref2 = 0;
unsigned long cmis0 = 0, cmis1 = 0, cmis2 = 0;
#endif
// Do warmup iterations
bool isok = true;
for (int32_t i = 0; i < warmup_iterations; i++) {
if (verbose) {
std::cout << "[verbose] warmup iteration # " << i << std::endl;
}
simdjson::ParsedJson pj;
bool allocok = pj.allocate_capacity(p.size());
if (!allocok) {
std::cerr << "failed to allocate memory" << std::endl;
return EXIT_FAILURE;
}
isok = (simdjson::stage1_ptr((const uint8_t *)p.data(), p.size(), pj) ==
simdjson::SUCCESS);
isok = isok &&
(simdjson::SUCCESS ==
simdjson::unified_ptr((const uint8_t *)p.data(), p.size(), pj));
if (!isok) {
std::cerr << pj.get_error_message() << std::endl;
std::cerr << "Could not parse. " << std::endl;
return EXIT_FAILURE;
}
}
#ifndef SQUASH_COUNTERS
for (uint32_t i = 0; i < iterations; i++) {
for (int32_t i = 0; i < iterations; i++) {
if (verbose) {
std::cout << "[verbose] iteration # " << i << std::endl;
}
@ -265,8 +307,9 @@ int main(int argc, char *argv[]) {
}
}
#endif
// we do it again, this time just measuring the elapsed time
for (uint32_t i = 0; i < iterations; i++) {
for (int32_t i = 0; i < iterations; i++) {
if (verbose) {
std::cout << "[verbose] iteration # " << i << std::endl;
}

73
benchmark/perfdiff.cpp Normal file
View File

@ -0,0 +1,73 @@
#include <cstdio>
#include <iostream>
#include <memory>
#include <stdexcept>
#include <string>
#include <sstream>
#include <array>
#ifdef _WIN32
#define popen _popen
#define pclose _pclose
#endif
std::string exec(const char* cmd) {
std::array<char, 128> buffer;
std::string result;
std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd, "r"), pclose);
if (!pipe) {
throw std::runtime_error("popen() failed!");
}
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
result += buffer.data();
}
return result;
}
double readThroughput(std::string parseOutput) {
std::istringstream output(parseOutput);
std::string line;
double result = 0;
int numResults = 0;
while (std::getline(output, line)) {
int pos = 0;
for (int i=0; i<5; i++) {
pos = line.find('\t', pos);
if (pos < 0) {
std::cerr << "Command printed out a line with less than 5 fields in it:\n" << line << std::endl;
}
pos++;
}
result += std::stod(line.substr(pos));
numResults++;
}
return result / numResults;
}
const double ERROR_MARGIN = 10; // 10%
const double INTERLEAVED_ATTEMPTS = 4;
int main(int argc, char *argv[]) {
if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <new parse cmd> <reference parse cmd>";
return 1;
}
double newThroughput = 0;
double referenceThroughput = 0;
for (int attempt=0; attempt < INTERLEAVED_ATTEMPTS; attempt++) {
newThroughput += readThroughput(exec(argv[1]));
referenceThroughput += readThroughput(exec(argv[2]));
}
newThroughput /= INTERLEAVED_ATTEMPTS;
referenceThroughput /= INTERLEAVED_ATTEMPTS;
std::cout << "New throughput: " << newThroughput << std::endl;
std::cout << "Ref throughput: " << referenceThroughput << std::endl;
double percentDifference = ((newThroughput / referenceThroughput) - 1.0) * 100;
std::cout << "Difference: " << percentDifference << "%" << std::endl;
if (percentDifference < -ERROR_MARGIN) {
std::cerr << "New throughput is more than " << ERROR_MARGIN << "% degraded from reference throughput!" << std::endl;
return 1;
}
return 0;
}

37
include/simdjson/common_defs.h
View File

@ -43,38 +43,9 @@
#define unlikely(x) x
#endif
// For Visual Studio compilers, same-page buffer overrun is not fine.
#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false
#else
// For non-Visual Studio compilers, we may assume that same-page buffer overrun
// is fine. However, it will make it difficult to be "valgrind clean".
//#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN
//#define ALLOW_SAME_PAGE_BUFFER_OVERRUN true
//#else
#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false
//#endif
// The following is likely unnecessarily complex.
#ifdef __SANITIZE_ADDRESS__
// we have GCC, stuck with https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368
#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false
#elif defined(__has_feature)
// we have CLANG?
// todo: if we're setting ALLOW_SAME_PAGE_BUFFER_OVERRUN to false, why do we
// have a non-empty qualifier?
#if (__has_feature(address_sanitizer))
#define ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER \
__attribute__((no_sanitize("address")))
#endif
#endif
#if defined(__has_feature)
#if (__has_feature(memory_sanitizer))
#define LENIENT_MEM_SANITIZER __attribute__((no_sanitize("memory")))
#endif
#endif
#define really_inline inline __attribute__((always_inline, unused))
#define never_inline inline __attribute__((noinline, unused))
@ -91,12 +62,4 @@
#endif // MSC_VER
// if it does not apply, make it an empty macro
#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER
#define ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER
#endif
#ifndef LENIENT_MEM_SANITIZER
#define LENIENT_MEM_SANITIZER
#endif
#endif // SIMDJSON_COMMON_DEFS_H

26
include/simdjson/jsonparser.h
View File

@ -29,38 +29,12 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj,
}
bool reallocated = false;
if (realloc_if_needed) {
#if ALLOW_SAME_PAGE_BUFFER_OVERRUN
// realloc is needed if the end of the memory crosses a page
#ifdef _MSC_VER
SYSTEM_INFO sysInfo;
GetSystemInfo(&sysInfo);
long page_size = sysInfo.dwPageSize;
#else
long page_size = sysconf(_SC_PAGESIZE);
#endif
//////////////
// We want to check that buf + len - 1 and buf + len - 1 + SIMDJSON_PADDING
// are in the same page.
// That is, we want to check that
// (buf + len - 1) / page_size == (buf + len - 1 + SIMDJSON_PADDING) /
// page_size That's true if (buf + len - 1) % page_size + SIMDJSON_PADDING <
// page_size.
///////////
if ((reinterpret_cast<uintptr_t>(buf + len - 1) % page_size) +
SIMDJSON_PADDING <
static_cast<uintptr_t>(page_size)) {
#else // SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN
if (true) { // if not SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN, we always
// reallocate
#endif
const uint8_t *tmp_buf = buf;
buf = (uint8_t *)allocate_padded_buffer(len);
if (buf == NULL)
return simdjson::MEMALLOC;
memcpy((void *)buf, tmp_buf, len);
reallocated = true;
} // if (true) OR if ( (reinterpret_cast<uintptr_t>(buf + len - 1) %
// page_size ) + SIMDJSON_PADDING < static_cast<uintptr_t>(page_size) ) {
} // if(realloc_if_needed) {
int stage1_is_ok = simdjson::find_structural_bits<T>(buf, len, pj);
if (stage1_is_ok != simdjson::SUCCESS) {

2
include/simdjson/stage2_build_tape.h
View File

@ -72,7 +72,7 @@ really_inline bool is_valid_null_atom(const uint8_t *loc) {
}
template <Architecture T = Architecture::NATIVE>
WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER int
WARN_UNUSED int
unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);
template <Architecture T = Architecture::NATIVE>

5
include/simdjson/stage2_build_tape_common.h
View File

@ -42,12 +42,9 @@ namespace simdjson {
* for documentation.
***********/
template <>
WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER int
WARN_UNUSED int
unified_machine<TARGETED_ARCHITECTURE>(const uint8_t *buf, size_t len,
ParsedJson &pj) {
if (ALLOW_SAME_PAGE_BUFFER_OVERRUN) {
memset((uint8_t *)buf + len, 0, SIMDJSON_PADDING); /* to please valgrind */
}
uint32_t i = 0; /* index of the structural character (0,1,2,3...) */
uint32_t idx; /* location of the structural character in the input (buf) */
uint8_t c; /* used to track the (structural) character we are looking at,

2
include/simdjson/stringparsing.h
View File

@ -90,7 +90,7 @@ parse_string_helper find_bs_bits_and_quote_bits(const uint8_t *src,
uint8_t *dst);
template <Architecture T>
WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER
WARN_UNUSED
really_inline bool
parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, ParsedJson &pj,
UNUSED const uint32_t depth, UNUSED uint32_t offset);

2
include/simdjson/stringparsing_common.h
View File

@ -10,7 +10,7 @@ TARGETED_REGION
namespace simdjson {
template <>
WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER
WARN_UNUSED
really_inline bool
parse_string<TARGETED_ARCHITECTURE>(UNUSED const uint8_t *buf,
UNUSED size_t len, ParsedJson &pj,

30
scripts/checkperf.sh Normal file
View File

@ -0,0 +1,30 @@
#!/bin/bash
set -e
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
# Arguments: perfdiff.sh <branch> <test json files>
if [ -z "$1" ]; then reference_branch="master"; else reference_branch=$1; shift; fi
if [ -z "$*" ]; then perftests="jsonexamples/twitter.json"; else perftests=$*; fi
# Clone and build the reference branch's parse
echo "Cloning and build the reference branch ($reference_branch) ..."
reference=$current/benchbranch/$reference_branch
rm -rf $reference
mkdir -p $reference
git clone --depth 1 -b $reference_branch https://github.com/lemire/simdjson $reference
cd $reference
make parse
# Build the current branch's parse
echo "Building the current branch ..."
current=$SCRIPTPATH/..
cd $current
make clean
make parse
# Run them and diff performance
make perfdiff
echo "Running perfdiff:"
./perfdiff "$current/parse -t $perftests" "$reference/parse -t $perftests"

18
src/jsonparser.cpp
View File

@ -12,21 +12,21 @@ namespace simdjson {
// function pointer type for json_parse
using json_parse_functype = int(const uint8_t *buf, size_t len, ParsedJson &pj,
bool realloc_if_needed);
bool realloc);
// Pointer that holds the json_parse implementation corresponding to the
// available SIMD instruction set
extern std::atomic<json_parse_functype *> json_parse_ptr;
int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj,
bool realloc_if_needed) {
return json_parse_ptr.load(std::memory_order_relaxed)(buf, len, pj, realloc_if_needed);
bool realloc) {
return json_parse_ptr.load(std::memory_order_relaxed)(buf, len, pj, realloc);
}
int json_parse(const char *buf, size_t len, ParsedJson &pj,
bool realloc_if_needed) {
bool realloc) {
return json_parse_ptr.load(std::memory_order_relaxed)(reinterpret_cast<const uint8_t *>(buf), len, pj,
realloc_if_needed);
realloc);
}
Architecture find_best_supported_implementation() {
@ -50,7 +50,7 @@ Architecture find_best_supported_implementation() {
// Responsible to select the best json_parse implementation
int json_parse_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj,
bool realloc_if_needed) {
bool realloc) {
Architecture best_implementation = find_best_supported_implementation();
// Selecting the best implementation
switch (best_implementation) {
@ -72,18 +72,18 @@ int json_parse_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj,
return simdjson::UNEXPECTED_ERROR;
}
return json_parse_ptr.load(std::memory_order_relaxed)(buf, len, pj, realloc_if_needed);
return json_parse_ptr.load(std::memory_order_relaxed)(buf, len, pj, realloc);
}
std::atomic<json_parse_functype *> json_parse_ptr = &json_parse_dispatch;
WARN_UNUSED
ParsedJson build_parsed_json(const uint8_t *buf, size_t len,
bool realloc_if_needed) {
bool realloc) {
ParsedJson pj;
bool ok = pj.allocate_capacity(len);
if (ok) {
json_parse(buf, len, pj, realloc_if_needed);
json_parse(buf, len, pj, realloc);
} else {
std::cerr << "failure during memory allocation " << std::endl;
}

31
tools/cmake/FindOptions.cmake
View File

@ -12,19 +12,26 @@ if(SIMDJSON_SANITIZE)
endif()
endif()
if (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)|(x86_64)")
set (X64 TRUE)
else ()
set (X64 FALSE)
endif ()
if(SIMDJSON_DISABLE_AVX)
if(NOT MSVC)
set (OPT_FLAGS "${OPT_FLAGS} -mno-avx -mno-bmi -mno-pclmul -msse4.2")
else()
set (OPT_FLAGS "${OPT_FLAGS}")
endif()
else()
# some compilers like clang do not automagically define __AVX2__ and __BMI2__ even when the hardware supports it
if(NOT MSVC)
#set (OPT_FLAGS "${OPT_FLAGS} -mpclmul")
else()
#set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2")
if(X64)
message(STATUS "Detected x64 processor: ${CMAKE_SYSTEM_PROCESSOR}")
if(SIMDJSON_DISABLE_AVX)
message(STATUS "Disabling AVX. No runtime dispatch.")
if(NOT MSVC)
set (OPT_FLAGS "${OPT_FLAGS} -mno-avx -mno-bmi -mpclmul -msse4.2") #westmere
else()
set (OPT_FLAGS "${OPT_FLAGS}")
endif()
else()
if(NOT MSVC)
message(STATUS "Assuming Westmere or better")
set (OPT_FLAGS "${OPT_FLAGS} -mpclmul -msse4.2")
endif()
endif()
endif()