Merge branch 'master' into jkeiser/cmake-fuzz-noexceptions

This commit is contained in:
Daniel Lemire 2020-04-23 10:29:14 -04:00 committed by GitHub
commit 7a2fda891c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
69 changed files with 892 additions and 603 deletions

View File

@ -12,14 +12,19 @@ environment:
matrix:
- SIMDJSON_BUILD_STATIC: ON
SIMDJSON_ENABLE_THREADS: OFF
SIMDJSON_PLATFORM: x64
- SIMDJSON_BUILD_STATIC: ON
SIMDJSON_ENABLE_THREADS: OFF
SIMDJSON_PLATFORM: Win32
- SIMDJSON_BUILD_STATIC: OFF
SIMDJSON_ENABLE_THREADS: ON
SIMDJSON_PLATFORM: x64
build_script:
- set
- mkdir build
- cd build
- cmake -DSIMDJSON_BUILD_STATIC=%SIMDJSON_BUILD_STATIC% -DSIMDJSON_ENABLE_THREADS=%SIMDJSON_ENABLE_THREADS% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_GENERATOR_PLATFORM=x64 ..
- cmake -DSIMDJSON_BUILD_STATIC=%SIMDJSON_BUILD_STATIC% -DSIMDJSON_ENABLE_THREADS=%SIMDJSON_ENABLE_THREADS% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_GENERATOR_PLATFORM=%SIMDJSON_PLATFORM% ..
- cmake -LH ..
- cmake --build . --config %Configuration% --verbose
@ -29,7 +34,7 @@ test_script:
matrix:
fast_finish: true
exclude:
# Don't build all variants on 2019, just running it to make sure readme_tests succeed
- image: Visual Studio 2019
# Don't build all variants on 2017, just running it to make sure readme_tests succeed
- image: Visual Studio 2017
SIMDJSON_BUILD_STATIC: ON
SIMDJSON_ENABLE_THREADS: OFF

View File

@ -25,6 +25,13 @@ executors:
CXX: g++
BUILD_FLAGS: -j
CTEST_FLAGS: -j4 --output-on-failure
clang9:
docker:
- image: conanio/clang9
environment:
CXX: clang++-9
BUILD_FLAGS: -j
CTEST_FLAGS: -j4 --output-on-failure
clang6:
docker:
@ -50,10 +57,13 @@ commands:
- run: make test
- run: make checkperf
cmake_prep:
install_cmake:
steps:
- run: apt-get update -qq
- run: apt-get install -y cmake
cmake_prep:
steps:
- checkout
cmake_build:
@ -78,6 +88,7 @@ commands:
- run: SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
- run: ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now.
# we not only want cmake to build and run tests, but we want also a successful installation from which we can build, link and run programs
cmake_install_test: # this version builds, install, test and then verify from the installation
steps:
@ -91,60 +102,65 @@ jobs:
description: Build and run tests on GCC 7 and AVX 2 with a cmake static build
executor: gcc7
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
steps: [ cmake_test_all, cmake_install_test ]
steps: [ install_cmake, cmake_test_all, cmake_install_test ]
clang6:
description: Build and run tests on clang 6 and AVX 2 with a cmake static build
executor: clang6
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
steps: [ init_clang6, cmake_test_all, cmake_install_test ]
steps: [ init_clang6, install_cmake, cmake_test_all, cmake_install_test ]
# libcpp
libcpp-clang9:
description: Build and run tests on clang 6 and AVX 2 with a cmake static build and libc++
executor: clang9
environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON }
steps: [ cmake_test_all, cmake_install_test ]
# sanitize
sanitize-gcc9:
description: Build and run tests on GCC 9 and AVX 2 with a cmake sanitize build
executor: gcc9
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
steps: [ cmake_test_all ]
steps: [ install_cmake, cmake_test_all ]
sanitize-clang6:
description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build
executor: clang6
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
steps: [ init_clang6, cmake_test_all ]
steps: [ init_clang6, install_cmake, cmake_test_all ]
# dynamic
dynamic-gcc7:
description: Build and run tests on GCC 7 and AVX 2 with a cmake dynamic build
executor: gcc7
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
steps: [ cmake_test, cmake_install_test ]
steps: [ install_cmake, cmake_test, cmake_install_test ]
dynamic-clang6:
description: Build and run tests on clang 6 and AVX 2 with a cmake dynamic build
executor: clang6
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
steps: [ init_clang6, cmake_test, cmake_install_test ]
steps: [ init_clang6, install_cmake, cmake_test, cmake_install_test ]
# unthreaded
unthreaded-gcc7:
description: Build and run tests on GCC 7 and AVX 2 *without* threads
executor: gcc7
environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
steps: [ cmake_test, cmake_install_test ]
steps: [ install_cmake, cmake_test, cmake_install_test ]
unthreaded-clang6:
description: Build and run tests on Clang 6 and AVX 2 *without* threads
executor: clang6
environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
steps: [ init_clang6, cmake_test, cmake_install_test ]
steps: [ init_clang6, install_cmake, cmake_test, cmake_install_test ]
# noexcept
noexcept-gcc7:
description: Build and run tests on GCC 7 and AVX 2 with exceptions off
executor: gcc7
environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
steps: [ cmake_test, cmake_install_test ]
steps: [ install_cmake, cmake_test, cmake_install_test ]
noexcept-clang6:
description: Build and run tests on GCC 7 and AVX 2 with exceptions off
executor: clang6
environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
steps: [ init_clang6, cmake_test, cmake_install_test ]
steps: [ init_clang6, install_cmake, cmake_test, cmake_install_test ]
#
# Misc.
@ -174,7 +190,10 @@ workflows:
# full multi-implementation tests
- gcc7
- clang6
# libc++
- libcpp-clang9
# full single-implementation tests
- sanitize-gcc9
- sanitize-clang6

21
.cirrus.yml Normal file
View File

@ -0,0 +1,21 @@
task:
timeout_in: 120m
freebsd_instance:
matrix:
- image_family: freebsd-13-0-snap
env:
ASSUME_ALWAYS_YES: YES
setup_script:
- pkg update -f
- pkg install bash
- pkg install cmake
- pkg install git
build_script:
- mkdir build
- cd build
- cmake ..
- make -j2
test_script:
- cd build
- ctest -j2 --output-on-failure -E checkperf

View File

@ -1,7 +1,7 @@
*
!.git
!Makefile
!amalgamation.sh
!amalgamate.sh
!benchmark
!dependencies
!include

View File

@ -47,7 +47,7 @@ platform:
steps:
- name: build
image: gcc:8
commands: [ make, make amalgamate ]
commands: [ make, make amalgamate_test ]
---
kind: pipeline
name: x64-slowtests
@ -87,7 +87,7 @@ steps:
image: gcc:8
environment:
EXTRA_FLAGS: -fno-exceptions
commands: [ make, make amalgamate ]
commands: [ make, make amalgamate_test ]
---
kind: pipeline
name: x64-noexceptions-slowtests
@ -139,7 +139,7 @@ platform:
steps:
- name: build
image: gcc:8
commands: [ make, make amalgamate ]
commands: [ make, make amalgamate_test ]
---
kind: pipeline
name: arm64-slowtests

9
.gitignore vendored
View File

@ -133,6 +133,11 @@ objs
/tests/allparserscheckfile
/tests/basictests
/tests/checkimplementation
/tests/compilation_failure_tests/dangling_parser_load_should_compile
/tests/compilation_failure_tests/dangling_parser_parse_padstring_should_compile
/tests/compilation_failure_tests/dangling_parser_parse_stdstring_should_compile
/tests/compilation_failure_tests/dangling_parser_parse_uchar_should_compile
/tests/compilation_failure_tests/dangling_parser_parse_uint8_should_compile
/tests/compilation_failure_tests/example_compiletest_should_compile
/tests/errortests
/tests/extracting_values_example
@ -213,3 +218,7 @@ _deps
# We check in a custom version of root Makefile that is not generated by CMake
!/Makefile
singleheader/amalgamation_demo
singleheader/amalgamation_demo.cpp
singleheader/simdjson.cpp
singleheader/simdjson.h

View File

@ -25,11 +25,13 @@ if(MSVC)
else()
option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON)
option(SIMDJSON_USE_LIBCPP "Use the libc++ library" OFF)
endif()
option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" OFF)
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake")
# We compile tools, tests, etc. with C++ 17. Override yourself if you need on a target.
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@ -38,6 +40,7 @@ set(CMAKE_MACOSX_RPATH OFF)
set(CMAKE_THREAD_PREFER_PTHREAD ON)
set(THREADS_PREFER_PTHREAD_FLAG ON)
# LTO seems to create all sorts of fun problems. Let us
# disable temporarily.
#include(CheckIPOSupported)
@ -52,12 +55,23 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
add_library(simdjson-flags INTERFACE)
if(MSVC)
target_compile_options(simdjson-flags INTERFACE /nologo /D_CRT_SECURE_NO_WARNINGS)
target_compile_options(simdjson-flags INTERFACE /WX /W3 /wd4005 /wd4996 /wd4267 /wd4244 /wd4113)
target_compile_options(simdjson-flags INTERFACE /WX /W3)
else()
target_compile_options(simdjson-flags INTERFACE -fPIC)
target_compile_options(simdjson-flags INTERFACE -Werror -Wall -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self)
target_compile_options(simdjson-flags INTERFACE -Werror -Wall -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self -Wconversion -Wno-sign-conversion)
endif()
if(SIMDJSON_USE_LIBCPP)
target_link_libraries(simdjson-flags INTERFACE -stdlib=libc++ -lc++abi)
# instead of the above line, we could have used
# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi")
# The next line is needed empirically.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
# we update CMAKE_SHARED_LINKER_FLAGS, this gets updated later as well
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lc++abi")
endif(SIMDJSON_USE_LIBCPP)
# Optional flags
option(SIMDJSON_IMPLEMENTATION_HASWELL "Include the haswell implementation" ON)
if(NOT SIMDJSON_IMPLEMENTATION_HASWELL)
@ -138,6 +152,15 @@ if (NOT MSVC)
add_custom_target(simdjson-user-cmakecache ALL DEPENDS ${SIMDJSON_USER_CMAKECACHE})
endif()
#
# Set up test data
#
enable_testing()
add_subdirectory(jsonchecker)
add_subdirectory(jsonexamples)
add_library(test-data INTERFACE)
target_link_libraries(test-data INTERFACE jsonchecker-data jsonexamples-data)
#
# Create the top level simdjson library (must be done at this level to use both src/ and include/
# directories) and tools
@ -146,16 +169,11 @@ add_subdirectory(include)
add_subdirectory(src)
add_subdirectory(windows)
add_subdirectory(tools)
add_subdirectory(singleheader)
#
# Compile tools / tests / benchmarks
#
enable_testing()
add_library(test-data INTERFACE)
target_compile_definitions(test-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/")
target_compile_definitions(test-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/")
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/twitter.json)
add_subdirectory(dependencies)
add_subdirectory(tests)

View File

@ -31,5 +31,6 @@ Paul Dreik
Jeremie Piotte
Matthew Wilson
Dušan Jovanović
Matjaž Ostroveršnik
# if you have contributed to the project and your name does not
# appear in this list, please let us know!

View File

@ -20,7 +20,7 @@ simdjson's source structure, from the top level, looks like this:
implementations).
* simdjson.cpp: A "master source" that includes all implementation files from src/. This is
equivalent to the distributed simdjson.cpp.
* arm64/|fallback/|haswell/|westmere/: Architecture-specific implementations. All functions are
* arm64/|fallback/|haswell/|westmere/: Architecture-specific implementations. All functions are
Each architecture defines its own namespace, e.g. simdjson::haswell.
* generic/: Generic implementations of the simdjson parser. These files may be included and
compiled multiple times, from whichever architectures use them. They assume they are already
@ -37,7 +37,7 @@ Other important files and directories:
* **.drone.yml:** Definitions for Drone CI.
* **.appveyor.yml:** Definitions for Appveyor CI (Windows).
* **.circleci:** Definitions for Circle CI.
* **amalgamation.sh:** Generates singleheader/simdjson.h and singleheader/simdjson.cpp for release.
* **amalgamate.sh:** Generates singleheader/simdjson.h and singleheader/simdjson.cpp for release.
* **benchmark:** This is where we do benchmarking. Benchmarking is core to every change we make; the
cardinal rule is don't regress performance without knowing exactly why, and what you're trading
for it. If you're not sure what else to do to check your performance, this is always a good start:
@ -56,7 +56,7 @@ Other important files and directories:
* **tools:** Source for executables that can be distributed with simdjson
> **Don't modify the files in singleheader/ directly; these are automatically generated.**
>
>
> While we distribute those files on release, we *maintain* the files under include/ and src/.
While simdjson distributes just two files from the singleheader/ directory, we *maintain* the code in
@ -73,7 +73,7 @@ you can regenerate them by running this at the top level:
make amalgamate
```
The amalgamator is at `amalgamation.sh` at the top level. It generates singleheader/simdjson.h by
The amalgamator is at `amalgamate.sh` at the top level. It generates singleheader/simdjson.h by
reading through include/simdjson.h, copy/pasting each header file into the amalgamated file at the
point it gets included (but only once per header). singleheader/simdjson.cpp is generated from
src/simdjson.cpp the same way, except files under generic/ may be included and copy/pasted multiple
@ -157,6 +157,19 @@ make
make test
```
linux way:
```
mkdir build # if necessary
cd build
cmake .. -DCMAKE_CXX_COMPILER=g++ # or
cmake .. -DCMAKE_CXX_COMPILER=clang++
make -j
```
### Usage (CMake on 64-bit Windows using Visual Studio)
We assume you have a common 64-bit Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later) or SSE 4.2 + CLMUL (2010 Westmere or later).
@ -189,7 +202,7 @@ On Windows (64-bit):
will build and install `simdjson` as a shared library.
```
.\vcpkg.exe install simdjson:x64-windows-static
.\vcpkg.exe install simdjson:x64-windows-static
```
will build and install `simdjson` as a static library.

View File

@ -178,13 +178,16 @@ quicktests: run_basictests run_quickstart readme_examples readme_examples_noexce
slowtests: run_testjson2json_sh run_issue150_sh
amalgamate:
./amalgamation.sh
singleheader/amalgamate.sh
singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamation_demo.cpp: amalgamation.sh src/simdjson.cpp $(SRCHEADERS) $(INCLUDEHEADERS)
./amalgamation.sh
singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp: singleheader/amalgamate.sh src/simdjson.cpp $(SRCHEADERS) $(INCLUDEHEADERS)
singleheader/amalgamate.sh
singleheader/demo: singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamation_demo.cpp
$(CXX) $(CXXFLAGS) -o singleheader/demo singleheader/amalgamation_demo.cpp -Isingleheader
singleheader/demo: singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp
$(CXX) $(CXXFLAGS) -o singleheader/demo singleheader/amalgamate_demo.cpp -Isingleheader
amalgamate_test: singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson
singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson
submodules:
-git submodule update --init --recursive

View File

@ -2,6 +2,7 @@
[![CircleCI](https://circleci.com/gh/simdjson/simdjson.svg?style=svg)](https://circleci.com/gh/simdjson/simdjson)
[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/simdjson.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&q=proj%3Asimdjson&can=2)
[![Build status](https://ci.appveyor.com/api/projects/status/ae77wp5v3lebmu6n/branch/master?svg=true)](https://ci.appveyor.com/project/lemire/simdjson-jmmti/branch/master)
[![CirrusCI](https://api.cirrus-ci.com/github/simdjson/simdjson.svg)](https://cirrus-ci.com/github/simdjson/simdjson)
[![][license img]][license]
simdjson : Parsing gigabytes of JSON per second

View File

@ -35,7 +35,7 @@ if (SIMDJSON_COMPETITION)
target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER)
endif()
if (NOT MSVC)
IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
add_test(NAME checkperf
COMMAND ${CMAKE_COMMAND} -E env
CHECKPERF_REPOSITORY=https://github.com/simdjson/simdjson
@ -46,4 +46,4 @@ if (NOT MSVC)
set_property(TEST checkperf APPEND PROPERTY LABELS per_implementation)
set_property(TEST checkperf APPEND PROPERTY DEPENDS parse perfdiff ${SIMDJSON_USER_CMAKECACHE})
set_property(TEST checkperf PROPERTY RUN_SERIAL TRUE)
endif()
ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")

View File

@ -60,7 +60,7 @@ uint64_t global_rdtsc_overhead = (uint64_t)UINT64_MAX;
do { \
uint64_t cycles_start, cycles_final, cycles_diff; \
uint64_t min_diff = UINT64_MAX; \
for (int i = 0; i < repeat; i++) { \
for (decltype(repeat) i = 0; i < repeat; i++) { \
__asm volatile("" ::: /* pretend to clobber */ "memory"); \
RDTSC_START(cycles_start); \
test; \
@ -73,7 +73,7 @@ uint64_t global_rdtsc_overhead = (uint64_t)UINT64_MAX;
} while (0)
double diff(timespec start, timespec end) {
return ((end.tv_nsec + 1000000000 * end.tv_sec) -
return static_cast<double>((end.tv_nsec + 1000000000 * end.tv_sec) -
(start.tv_nsec + 1000000000 * start.tv_sec)) /
1000000000.0;
}
@ -100,7 +100,7 @@ double diff(timespec start, timespec end) {
uint64_t sum_diff = 0; \
double sumclockdiff = 0; \
struct timespec time1, time2; \
for (int i = 0; i < repeat; i++) { \
for (decltype(repeat) i = 0; i < repeat; i++) { \
pre; \
__asm volatile("" ::: /* pretend to clobber */ "memory"); \
clock_gettime(CLOCK_REALTIME, &time1); \
@ -121,12 +121,12 @@ double diff(timespec start, timespec end) {
sum_diff += cycles_diff; \
} \
uint64_t S = size; \
float cycle_per_op = (min_diff) / (double)S; \
float avg_cycle_per_op = (sum_diff) / ((double)S * repeat); \
double cycle_per_op = static_cast<double>(min_diff) / static_cast<double>(S); \
double avg_cycle_per_op = static_cast<double>(sum_diff) / (static_cast<double>(S) * static_cast<double>(repeat)); \
double avg_gb_per_s = \
((double)S * repeat) / ((sumclockdiff)*1000.0 * 1000.0 * 1000.0); \
(static_cast<double>(S) * static_cast<double>(repeat)) / ((sumclockdiff)*1000.0 * 1000.0 * 1000.0); \
double max_gb_per_s = \
((double)S) / ((min_sumclockdiff)*1000.0 * 1000.0 * 1000.0); \
static_cast<double>(S) / (min_sumclockdiff * 1000.0 * 1000.0 * 1000.0); \
if (verbose) \
printf(" %7.3f %s per input byte (best) ", cycle_per_op, unitname); \
if (verbose) \
@ -137,7 +137,7 @@ double diff(timespec start, timespec end) {
if (verbose) \
printf(" %13.0f documents/s (best)", 1.0/min_sumclockdiff); \
if (verbose) \
printf(" %13.0f documents/s (avg)", 1.0/(sumclockdiff/repeat)); \
printf(" %13.0f documents/s (avg)", 1.0/(sumclockdiff/static_cast<double>(repeat))); \
if (!verbose) \
printf(" %20.3f %20.3f %20.3f %20.3f", cycle_per_op, \
avg_cycle_per_op - cycle_per_op, max_gb_per_s, \
@ -170,8 +170,8 @@ double diff(timespec start, timespec end) {
sum_diff += cycles_diff; \
} \
uint64_t S = size; \
float cycle_per_op = (min_diff) / (double)S; \
float avg_cycle_per_op = (sum_diff) / ((double)S * repeat); \
double cycle_per_op = static_cast<double>(min_diff) / static_cast<double>(S); \
double avg_cycle_per_op = static_cast<double>(sum_diff) / (static_cast<double>(S) * static_cast<double>(repeat)); \
if (verbose) \
printf(" %.3f %s per input byte (best) ", cycle_per_op, unitname); \
if (verbose) \

View File

@ -379,31 +379,31 @@ struct benchmarker {
printf("%s%-13s: %8.4f ns per block (%6.2f%%) - %8.4f ns per byte - %8.4f ns per structural - %8.3f GB/s\n",
prefix,
"Speed",
stage.elapsed_ns() / stats->blocks, // per block
100.0 * stage.elapsed_sec() / all_stages.elapsed_sec(), // %
stage.elapsed_ns() / stats->bytes, // per byte
stage.elapsed_ns() / stats->structurals, // per structural
(json.size() / 1000000000.0) / stage.elapsed_sec() // GB/s
stage.elapsed_ns() / static_cast<double>(stats->blocks), // per block
percent(stage.elapsed_sec(), all_stages.elapsed_sec()), // %
stage.elapsed_ns() / static_cast<double>(stats->bytes), // per byte
stage.elapsed_ns() / static_cast<double>(stats->structurals), // per structural
(static_cast<double>(json.size()) / 1000000000.0) / stage.elapsed_sec() // GB/s
);
if (collector.has_events()) {
printf("%s%-13s: %8.4f per block (%6.2f%%) - %8.4f per byte - %8.4f per structural - %8.3f GHz est. frequency\n",
prefix,
"Cycles",
stage.cycles() / stats->blocks,
100.0 * stage.cycles() / all_stages.cycles(),
stage.cycles() / stats->bytes,
stage.cycles() / stats->structurals,
stage.cycles() / static_cast<double>(stats->blocks),
percent(stage.cycles(), all_stages.cycles()),
stage.cycles() / static_cast<double>(stats->bytes),
stage.cycles() / static_cast<double>(stats->structurals),
(stage.cycles() / stage.elapsed_sec()) / 1000000000.0
);
printf("%s%-13s: %8.4f per block (%6.2f%%) - %8.4f per byte - %8.4f per structural - %8.3f per cycle\n",
prefix,
"Instructions",
stage.instructions() / stats->blocks,
100.0 * stage.instructions() / all_stages.instructions(),
stage.instructions() / stats->bytes,
stage.instructions() / stats->structurals,
stage.instructions() / stage.cycles()
stage.instructions() / static_cast<double>(stats->blocks),
percent(stage.instructions(), all_stages.instructions()),
stage.instructions() / static_cast<double>(stats->bytes),
stage.instructions() / static_cast<double>(stats->structurals),
stage.instructions() / static_cast<double>(stage.cycles())
);
// NOTE: removed cycles/miss because it is a somewhat misleading stat
@ -411,14 +411,21 @@ struct benchmarker {
prefix,
"Misses",
stage.branch_misses(),
100.0 * stage.branch_misses() / all_stages.branch_misses(),
percent(stage.branch_misses(), all_stages.branch_misses()),
stage.cache_misses(),
100.0 * stage.cache_misses() / all_stages.cache_misses(),
percent(stage.cache_misses(), all_stages.cache_misses()),
stage.cache_references()
);
}
}
static double percent(size_t a, size_t b) {
return 100.0 * static_cast<double>(a) / static_cast<double>(b);
}
static double percent(double a, double b) {
return 100.0 * a / b;
}
void print(bool tabbed_output, size_t iterations) const {
if (tabbed_output) {
char* filename_copy = (char*)malloc(strlen(filename)+1);
@ -432,14 +439,14 @@ struct benchmarker {
base[strlen(base)-5] = '\0';
}
double gb = json.size() / 1000000000.0;
double gb = static_cast<double>(json.size()) / 1000000000.0;
if (collector.has_events()) {
printf("\"%s\"\t%f\t%f\t%f\t%f\t%f\t%f\t%f\n",
base,
allocate_stage.best.cycles() / json.size(),
stage1.best.cycles() / json.size(),
stage2.best.cycles() / json.size(),
all_stages.best.cycles() / json.size(),
allocate_stage.best.cycles() / static_cast<double>(json.size()),
stage1.best.cycles() / static_cast<double>(json.size()),
stage2.best.cycles() / static_cast<double>(json.size()),
all_stages.best.cycles() / static_cast<double>(json.size()),
gb / all_stages.best.elapsed_sec(),
gb / stage1.best.elapsed_sec(),
gb / stage2.best.elapsed_sec());
@ -455,22 +462,22 @@ struct benchmarker {
printf("\n");
printf("%s\n", filename);
printf("%s\n", string(strlen(filename), '=').c_str());
printf("%9zu blocks - %10zu bytes - %5zu structurals (%5.1f %%)\n", stats->bytes / BYTES_PER_BLOCK, stats->bytes, stats->structurals, 100.0 * stats->structurals / stats->bytes);
printf("%9zu blocks - %10zu bytes - %5zu structurals (%5.1f %%)\n", stats->bytes / BYTES_PER_BLOCK, stats->bytes, stats->structurals, percent(stats->structurals, stats->bytes));
if (stats) {
printf("special blocks with: utf8 %9zu (%5.1f %%) - escape %9zu (%5.1f %%) - 0 structurals %9zu (%5.1f %%) - 1+ structurals %9zu (%5.1f %%) - 8+ structurals %9zu (%5.1f %%) - 16+ structurals %9zu (%5.1f %%)\n",
stats->blocks_with_utf8, 100.0 * stats->blocks_with_utf8 / stats->blocks,
stats->blocks_with_escapes, 100.0 * stats->blocks_with_escapes / stats->blocks,
stats->blocks_with_0_structurals, 100.0 * stats->blocks_with_0_structurals / stats->blocks,
stats->blocks_with_1_structural, 100.0 * stats->blocks_with_1_structural / stats->blocks,
stats->blocks_with_8_structurals, 100.0 * stats->blocks_with_8_structurals / stats->blocks,
stats->blocks_with_16_structurals, 100.0 * stats->blocks_with_16_structurals / stats->blocks);
stats->blocks_with_utf8, percent(stats->blocks_with_utf8, stats->blocks),
stats->blocks_with_escapes, percent(stats->blocks_with_escapes, stats->blocks),
stats->blocks_with_0_structurals, percent(stats->blocks_with_0_structurals, stats->blocks),
stats->blocks_with_1_structural, percent(stats->blocks_with_1_structural, stats->blocks),
stats->blocks_with_8_structurals, percent(stats->blocks_with_8_structurals, stats->blocks),
stats->blocks_with_16_structurals, percent(stats->blocks_with_16_structurals, stats->blocks));
printf("special block flips: utf8 %9zu (%5.1f %%) - escape %9zu (%5.1f %%) - 0 structurals %9zu (%5.1f %%) - 1+ structurals %9zu (%5.1f %%) - 8+ structurals %9zu (%5.1f %%) - 16+ structurals %9zu (%5.1f %%)\n",
stats->blocks_with_utf8_flipped, 100.0 * stats->blocks_with_utf8_flipped / stats->blocks,
stats->blocks_with_escapes_flipped, 100.0 * stats->blocks_with_escapes_flipped / stats->blocks,
stats->blocks_with_0_structurals_flipped, 100.0 * stats->blocks_with_0_structurals_flipped / stats->blocks,
stats->blocks_with_1_structural_flipped, 100.0 * stats->blocks_with_1_structural_flipped / stats->blocks,
stats->blocks_with_8_structurals_flipped, 100.0 * stats->blocks_with_8_structurals_flipped / stats->blocks,
stats->blocks_with_16_structurals_flipped, 100.0 * stats->blocks_with_16_structurals_flipped / stats->blocks);
stats->blocks_with_utf8_flipped, percent(stats->blocks_with_utf8_flipped, stats->blocks),
stats->blocks_with_escapes_flipped, percent(stats->blocks_with_escapes_flipped, stats->blocks),
stats->blocks_with_0_structurals_flipped, percent(stats->blocks_with_0_structurals_flipped, stats->blocks),
stats->blocks_with_1_structural_flipped, percent(stats->blocks_with_1_structural_flipped, stats->blocks),
stats->blocks_with_8_structurals_flipped, percent(stats->blocks_with_8_structurals_flipped, stats->blocks),
stats->blocks_with_16_structurals_flipped, percent(stats->blocks_with_16_structurals_flipped, stats->blocks));
}
printf("\n");
printf("All Stages\n");
@ -497,7 +504,7 @@ struct benchmarker {
freqmin, freqmax, freqall);
}
}
printf("\n%.1f documents parsed per second\n", iterations/loop.best.elapsed_sec());
printf("\n%.1f documents parsed per second\n", static_cast<double>(iterations)/static_cast<double>(loop.best.elapsed_sec()));
}
}
};

View File

@ -366,7 +366,7 @@ int main(int argc, char *argv[]) {
size_t size = s1.size();
int repeat = 500;
int volume = p.size();
size_t volume = p.size();
if (just_data) {
printf(
"name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");

View File

@ -56,11 +56,11 @@ struct event_count {
double elapsed_sec() const { return duration<double>(elapsed).count(); }
double elapsed_ns() const { return duration<double, std::nano>(elapsed).count(); }
double cycles() const { return event_counts[CPU_CYCLES]; }
double instructions() const { return event_counts[INSTRUCTIONS]; }
double branch_misses() const { return event_counts[BRANCH_MISSES]; }
double cache_references() const { return event_counts[CACHE_REFERENCES]; }
double cache_misses() const { return event_counts[CACHE_MISSES]; }
double cycles() const { return static_cast<double>(event_counts[CPU_CYCLES]); }
double instructions() const { return static_cast<double>(event_counts[INSTRUCTIONS]); }
double branch_misses() const { return static_cast<double>(event_counts[BRANCH_MISSES]); }
double cache_references() const { return static_cast<double>(event_counts[CACHE_REFERENCES]); }
double cache_misses() const { return static_cast<double>(event_counts[CACHE_MISSES]); }
event_count& operator=(const event_count& other) {
this->elapsed = other.elapsed;

View File

@ -12,7 +12,7 @@ double bench(std::string filename, simdjson::padded_string& p) {
std::chrono::time_point<std::chrono::steady_clock> end_clock =
std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed = end_clock - start_clock;
return (p.size() / (1024. * 1024 * 1024.)) / elapsed.count();
return (static_cast<double>(p.size()) / (1024. * 1024. * 1024.)) / elapsed.count();
}
int main(int argc, char *argv[]) {
@ -32,7 +32,7 @@ int main(int argc, char *argv[]) {
double meanval = 0;
double maxval = 0;
double minval = 10000;
std::cout << "file size: "<< (p.size() / (1024. * 1024 * 1024.)) << " GB" <<std::endl;
std::cout << "file size: "<< (static_cast<double>(p.size()) / (1024. * 1024. * 1024.)) << " GB" <<std::endl;
size_t times = p.size() > 1024*1024*1024 ? 5 : 50;
#if __cpp_exceptions
try {
@ -49,7 +49,7 @@ std::cout << "file size: "<< (p.size() / (1024. * 1024 * 1024.)) << " GB" <<std
return EXIT_FAILURE;
}
#endif
std::cout << "average speed: " << meanval / times << " GB/s"<< std::endl;
std::cout << "average speed: " << meanval / static_cast<double>(times) << " GB/s"<< std::endl;
std::cout << "min speed : " << minval << " GB/s" << std::endl;
std::cout << "max speed : " << maxval << " GB/s" << std::endl;
return EXIT_SUCCESS;

View File

@ -18,7 +18,7 @@ template <int TYPE = PERF_TYPE_HARDWARE> class LinuxEvents {
int fd;
bool working;
perf_event_attr attribs;
int num_events;
size_t num_events;
std::vector<uint64_t> temp_result_vec;
std::vector<uint64_t> ids;
@ -43,7 +43,7 @@ public:
uint32_t i = 0;
for (auto config : config_vec) {
attribs.config = config;
fd = syscall(__NR_perf_event_open, &attribs, pid, cpu, group, flags);
fd = static_cast<int>(syscall(__NR_perf_event_open, &attribs, pid, cpu, group, flags));
if (fd == -1) {
report_error("perf_event_open");
}

View File

@ -87,7 +87,7 @@ int main(int argc, char *argv[]) {
buffer[p.size()] = '\0';
int repeat = 50;
int volume = p.size();
size_t volume = p.size();
if (just_data) {
printf(
"name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
@ -113,7 +113,7 @@ int main(int argc, char *argv[]) {
printf("minisize = %zu, original size = %zu (minified down to %.2f percent "
"of original) \n",
outlength, p.size(), outlength * 100.0 / p.size());
outlength, p.size(), static_cast<double>(outlength) * 100.0 / static_cast<double>(p.size()));
/***
* Is it worth it to minify before parsing?

View File

@ -56,7 +56,7 @@ int main (int argc, char *argv[]){
auto end = std::chrono::steady_clock::now();
std::chrono::duration<double> secs = end - start;
double speedinGBs = (p.size()) / (secs.count() * 1000000000.0);
double speedinGBs = static_cast<double>(p.size()) / (static_cast<double>(secs.count()) * 1000000000.0);
std::cout << speedinGBs << "\t\t\t\t" << count << std::endl;
if (parse_res != simdjson::SUCCESS) {
@ -88,7 +88,7 @@ int main (int argc, char *argv[]){
auto end = std::chrono::steady_clock::now();
std::chrono::duration<double> secs = end - start;
double speedinGBs = (p.size()) / (secs.count() * 1000000000.0);
double speedinGBs = static_cast<double>(p.size()) / (static_cast<double>(secs.count()) * 1000000000.0);
if (speedinGBs > batch_size_res.at(i))
batch_size_res[i] = speedinGBs;
@ -136,7 +136,7 @@ int main (int argc, char *argv[]){
}
double min_result = *min_element(res.begin(), res.end());
double speedinGBs = (p.size()) / (min_result * 1000000000.0);
double speedinGBs = static_cast<double>(p.size()) / (min_result * 1000000000.0);
std::cout << "Min: " << min_result << " bytes read: " << p.size()

View File

@ -336,7 +336,7 @@ int main(int argc, char *argv[]) {
assert(stat_equal(s1, s2));
assert(stat_equal(s1, s3));
int repeat = 50;
int volume = p.size();
size_t volume = p.size();
if (just_data) {
printf("name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
}

View File

@ -77,14 +77,14 @@ never_inline size_t sum_line_lengths(char * data, size_t length) {
}
bool bench(const char *filename, bool verbose, bool just_data, int repeat_multiplier) {
bool bench(const char *filename, bool verbose, bool just_data, double repeat_multiplier) {
auto [p, err] = simdjson::padded_string::load(filename);
if (err) {
std::cerr << "Could not load the file " << filename << std::endl;
return false;
}
int repeat = (50000000 * repeat_multiplier) / p.size();
int repeat = static_cast<int>((50000000 * repeat_multiplier) / static_cast<double>(p.size()));
if (repeat < 10) { repeat = 10; }
if (verbose) {
@ -97,7 +97,7 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
std::cout << p.size() << " B";
std::cout << ": will run " << repeat << " iterations." << std::endl;
}
int volume = p.size();
size_t volume = p.size();
if (just_data) {
printf("%-42s %20s %20s %20s %20s \n", "name", "cycles_per_byte",
"cycles_per_byte_err", "gb_per_s", "gb_per_s_err");
@ -214,7 +214,7 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
buffer[p.size()] = '\0';
BEST_TIME(
"jsmn ",
(jsmn_parse(&jparser, buffer, p.size(), tokens.get(), p.size()) > 0),
(jsmn_parse(&jparser, buffer, p.size(), tokens.get(), static_cast<unsigned int>(p.size())) > 0),
true, jsmn_init(&jparser), repeat, volume, !just_data);
}
memcpy(buffer, p.data(), p.size());
@ -253,7 +253,7 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
results.resize(evts.size());
stats.resize(evts.size());
std::fill(stats.begin(), stats.end(), 0); // unnecessary
for (int i = 0; i < repeat; i++) {
for (decltype(repeat) i = 0; i < repeat; i++) {
unified.start();
auto parse_error = parser.parse(p).error();
if (parse_error)
@ -265,13 +265,13 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
printf("simdjson : cycles %10.0f instructions %10.0f branchmisses %10.0f "
"cacheref %10.0f cachemisses %10.0f bytespercachemiss %10.0f "
"inspercycle %10.1f insperbyte %10.1f\n",
stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
static_cast<double>(stats[0]) / static_cast<double>(repeat), static_cast<double>(stats[1]) / static_cast<double>(repeat),
static_cast<double>(stats[2]) / static_cast<double>(repeat), static_cast<double>(stats[3]) / static_cast<double>(repeat),
static_cast<double>(stats[4]) / static_cast<double>(repeat), static_cast<double>(volume) * static_cast<double>(repeat) / static_cast<double>(stats[2]),
static_cast<double>(stats[1]) / static_cast<double>(stats[0]), static_cast<double>(stats[1]) / (static_cast<double>(volume) * static_cast<double>(repeat)));
std::fill(stats.begin(), stats.end(), 0);
for (int i = 0; i < repeat; i++) {
for (decltype(repeat) i = 0; i < repeat; i++) {
memcpy(buffer, p.data(), p.size());
buffer[p.size()] = '\0';
unified.start();
@ -285,13 +285,13 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
printf("RapidJSON: cycles %10.0f instructions %10.0f branchmisses %10.0f "
"cacheref %10.0f cachemisses %10.0f bytespercachemiss %10.0f "
"inspercycle %10.1f insperbyte %10.1f\n",
stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
static_cast<double>(stats[0]) / static_cast<double>(repeat), static_cast<double>(stats[1]) / static_cast<double>(repeat),
static_cast<double>(stats[2]) / static_cast<double>(repeat), static_cast<double>(stats[3]) / static_cast<double>(repeat),
static_cast<double>(stats[4]) / static_cast<double>(repeat), static_cast<double>(volume) * static_cast<double>(repeat) / static_cast<double>(stats[2]),
static_cast<double>(stats[1]) / static_cast<double>(stats[0]), static_cast<double>(stats[1]) / (static_cast<double>(volume) * static_cast<double>(repeat)));
std::fill(stats.begin(), stats.end(), 0); // unnecessary
for (int i = 0; i < repeat; i++) {
for (decltype(repeat) i = 0; i < repeat; i++) {
memcpy(buffer, p.data(), p.size());
unified.start();
if (sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
@ -305,10 +305,10 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
printf("sajson : cycles %10.0f instructions %10.0f branchmisses %10.0f "
"cacheref %10.0f cachemisses %10.0f bytespercachemiss %10.0f "
"inspercycle %10.1f insperbyte %10.1f\n",
stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
static_cast<double>(stats[0]) / static_cast<double>(repeat), static_cast<double>(stats[1]) / static_cast<double>(repeat),
static_cast<double>(stats[2]) / static_cast<double>(repeat), static_cast<double>(stats[3]) / static_cast<double>(repeat),
static_cast<double>(stats[4]) / static_cast<double>(repeat), static_cast<double>(volume) * static_cast<double>(repeat) / static_cast<double>(stats[2]),
static_cast<double>(stats[1]) / static_cast<double>(stats[0]), static_cast<double>(stats[1]) / (static_cast<double>(volume) * static_cast<double>(repeat)));
}
#endif // __linux__

View File

@ -32,7 +32,7 @@ std::string exec(const char* cmd) {
std::cerr << "popen() failed!" << std::endl;
abort();
}
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
while (fgets(buffer.data(), int(buffer.size()), pipe.get()) != nullptr) {
result += buffer.data();
}
return result;
@ -44,10 +44,10 @@ double readThroughput(std::string parseOutput) {
double result = 0;
int numResults = 0;
while (std::getline(output, line)) {
int pos = 0;
std::string::size_type pos = 0;
for (int i=0; i<5; i++) {
pos = line.find('\t', pos);
if (pos < 0) {
if (pos == std::string::npos) {
std::cerr << "Command printed out a line with less than 5 fields in it:\n" << line << std::endl;
}
pos++;

View File

@ -199,8 +199,8 @@ int main(int argc, char *argv[]) {
std::cerr << "failure?" << std::endl;
}
}
printf("%f %f %f %f ", cy1 * 1.0 / iterations, cl1 * 1.0 / iterations,
cy2 * 1.0 / iterations, cl2 * 1.0 / iterations);
printf("%f %f %f %f ", static_cast<double>(cy1) / static_cast<double>(iterations), static_cast<double>(cl1) / static_cast<double>(iterations),
static_cast<double>(cy2) / static_cast<double>(iterations), static_cast<double>(cl2) / static_cast<double>(iterations));
#endif // __linux__
printf("\n");
return EXIT_SUCCESS;

View File

@ -20,7 +20,7 @@ Requirements
------------------
- A recent compiler (LLVM clang6 or better, GNU GCC 7 or better, Visual Studio 2017 or better). We require C++11 support as a minimum.
- A 64-bit system (ARM or x64 Intel/AMD). Under Visual Studio, you must compile for 64-bit (e.g., x64) targets.
- A 64-bit system (ARM or x64 Intel/AMD). We run tests on macOS, freeBSD, Linux and Windows. We recommend that Visual Studio users target a 64-bit build (x64).
Including simdjson
------------------

View File

@ -3,7 +3,13 @@
#
# TODO haven't quite decided the right way to run quickstart on Windows. Needs README update.
if (NOT MSVC)
#
# Note: on macOS and other platforms, the 'command' described below may not work even if the cmake builds.
# For example, it may be necessary to specify the sysroot, which CMake does, but the 'command' does not
# handle such niceties. On a case-by-case basis it is fixable but it requires work that CMake knows how
# to do but that is not trivial.
#
IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
# TODO run amalgamate first!
function(add_quickstart_test TEST_NAME SOURCE_FILE)
# Second argument is C++ standard name

View File

@ -105,6 +105,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
// gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary
#define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \
SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \
SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \
SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \
@ -121,11 +122,49 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
#endif // MSC_VER
//
// Backfill std::string_view using nonstd::string_view on C++11
//
#if (!SIMDJSON_CPLUSPLUS17)
// C++17 requires string_view.
#if SIMDJSON_CPLUSPLUS17
#define SIMDJSON_HAS_STRING_VIEW
#endif
// This macro (__cpp_lib_string_view) has to be defined
// for C++17 and better, but if it is otherwise defined,
// we are going to assume that string_view is available
// even if we do not have C++17 support.
#ifdef __cpp_lib_string_view
#define SIMDJSON_HAS_STRING_VIEW
#endif
// Some systems have string_view even if we do not have C++17 support,
// and even if __cpp_lib_string_view is undefined, it is the case
// with Apple clang version 11.
// We must handle it. *This is important.*
#ifndef SIMDJSON_HAS_STRING_VIEW
#if defined __has_include
// do not combine the next #if with the previous one (unsafe)
#if __has_include (<string_view>)
// now it is safe to trigger the include
#include <string_view> // though the file is there, it does not follow that we got the implementation
#if defined(_LIBCPP_STRING_VIEW)
// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceeded C++17,
// included string_view.
// This means that we have string_view *even though* we may not have C++17.
#define SIMDJSON_HAS_STRING_VIEW
#endif // _LIBCPP_STRING_VIEW
#endif // __has_include (<string_view>)
#endif // defined __has_include
#endif // def SIMDJSON_HAS_STRING_VIEW
// end of complicated but important routine to try to detect string_view.
//
// Backfill std::string_view using nonstd::string_view on systems where
// we expect that string_view is missing. Important: if we get this wrong,
// we will end up with two string_view definitions and potential trouble.
// That is why we work so hard above to avoid it.
//
#ifndef SIMDJSON_HAS_STRING_VIEW
SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
#include "simdjson/nonstd/string_view.hpp"
SIMDJSON_POP_DISABLE_WARNINGS
@ -133,6 +172,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
namespace std {
using string_view = nonstd::string_view;
}
#endif // if (SIMDJSON_CPLUSPLUS < 201703L)
#endif // SIMDJSON_HAS_STRING_VIEW
#undef SIMDJSON_HAS_STRING_VIEW // We are not going to need this macro anymore.
#endif // SIMDJSON_COMMON_DEFS_H

View File

@ -124,8 +124,9 @@ public:
* Get the next value.
*
* Part of the std::iterator interface.
*
*/
inline void operator++() noexcept;
inline iterator& operator++() noexcept;
/**
* Check if these values come from the same place in the JSON.
*
@ -205,8 +206,9 @@ public:
* Get the next key/value pair.
*
* Part of the std::iterator interface.
*
*/
inline void operator++() noexcept;
inline iterator& operator++() noexcept;
/**
* Check if these key value pairs come from the same place in the JSON.
*
@ -373,13 +375,13 @@ public:
bool dump_raw_tape(std::ostream &os) const noexcept;
/** @private Structural values. */
std::unique_ptr<uint64_t[]> tape;
std::unique_ptr<uint64_t[]> tape{};
/** @private String values.
*
* Should be at least byte_capacity.
*/
std::unique_ptr<uint8_t[]> string_buf;
std::unique_ptr<uint8_t[]> string_buf{};
private:
inline error_code allocate(size_t len) noexcept;
@ -660,8 +662,7 @@ public:
* to allocate an initial capacity, call allocate() after constructing the parser.
* Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process).
*/
really_inline parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept;
really_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept;
/**
* Take another parser's buffers and state.
*
@ -811,7 +812,7 @@ public:
* - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
* - other json errors if parsing fails.
*/
inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/**
* Parse a buffer containing many JSON documents.
@ -953,21 +954,21 @@ public:
/** @private Number of structural indices passed from stage 1 to stage 2 */
uint32_t n_structural_indexes{0};
/** @private Structural indices passed from stage 1 to stage 2 */
std::unique_ptr<uint32_t[]> structural_indexes;
std::unique_ptr<uint32_t[]> structural_indexes{};
/** @private Tape location of each open { or [ */
std::unique_ptr<scope_descriptor[]> containing_scope;
std::unique_ptr<scope_descriptor[]> containing_scope{};
#ifdef SIMDJSON_USE_COMPUTED_GOTO
/** @private Return address of each open { or [ */
std::unique_ptr<void*[]> ret_address;
std::unique_ptr<void*[]> ret_address{};
#else
/** @private Return address of each open { or [ */
std::unique_ptr<char[]> ret_address;
std::unique_ptr<char[]> ret_address{};
#endif
/** @private Next write location in the string buf for stage 2 parsing */
uint8_t *current_string_buf_loc;
uint8_t *current_string_buf_loc{};
/** @private Use `if (parser.parse(...).error())` instead */
bool valid{false};
@ -975,7 +976,7 @@ public:
error_code error{UNINITIALIZED};
/** @private Use `parser.parse(...).value()` instead */
document doc;
document doc{};
/** @private returns true if the document parsed was valid */
[[deprecated("Use the result of parser.parse() instead")]]

View File

@ -135,8 +135,8 @@ private:
error_code error{SUCCESS_AND_HAS_MORE};
#ifdef SIMDJSON_THREADS_ENABLED
error_code stage1_is_ok_thread{SUCCESS};
std::thread stage_1_thread;
dom::parser parser_thread;
std::thread stage_1_thread{};
dom::parser parser_thread{};
#endif
friend class dom::parser;
}; // class document_stream

View File

@ -17,6 +17,7 @@ namespace simdjson {
*/
class implementation {
public:
/**
* The name of this implementation.
*
@ -131,6 +132,7 @@ protected:
_required_instruction_sets(required_instruction_sets)
{
}
virtual ~implementation()=default;
private:
/**

View File

@ -213,7 +213,7 @@ inline error_code document::allocate(size_t capacity) noexcept {
// a pathological input like "[[[[..." would generate len tape elements, so
// need a capacity of at least len + 1, but it is also possible to do
// worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"
// worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"
//where len + 1 tape elements are
// generated, see issue https://github.com/lemire/simdjson/issues/345
size_t tape_capacity = ROUNDUP_N(capacity + 2, 64);
@ -229,12 +229,12 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
uint32_t string_length;
size_t tape_idx = 0;
uint64_t tape_val = tape[tape_idx];
uint8_t type = (tape_val >> 56);
uint8_t type = uint8_t(tape_val >> 56);
os << tape_idx << " : " << type;
tape_idx++;
size_t how_many = 0;
if (type == 'r') {
how_many = tape_val & internal::JSON_VALUE_MASK;
how_many = size_t(tape_val & internal::JSON_VALUE_MASK);
} else {
// Error: no starting root node?
return false;
@ -245,7 +245,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
os << tape_idx << " : ";
tape_val = tape[tape_idx];
payload = tape_val & internal::JSON_VALUE_MASK;
type = (tape_val >> 56);
type = uint8_t(tape_val >> 56);
switch (type) {
case '"': // we have a string
os << "string \"";
@ -312,7 +312,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
}
tape_val = tape[tape_idx];
payload = tape_val & internal::JSON_VALUE_MASK;
type = (tape_val >> 56);
type = uint8_t(tape_val >> 56);
os << tape_idx << " : " << type << "\t// pointing to " << payload
<< " (start root)\n";
return true;
@ -322,7 +322,9 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
// parser inline implementation
//
really_inline parser::parser(size_t max_capacity) noexcept
: _max_capacity{max_capacity}, loaded_bytes(nullptr, &aligned_free_char) {}
: _max_capacity{max_capacity},
loaded_bytes(nullptr, &aligned_free_char)
{}
inline bool parser::is_valid() const noexcept { return valid; }
inline int parser::get_error_code() const noexcept { return error; }
inline std::string parser::get_error_message() const noexcept { return error_message(error); }
@ -471,7 +473,7 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept {
//
// Initialize stage 1 output
//
uint32_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7;
size_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7;
structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); // TODO realloc
if (!structural_indexes) {
return MEMALLOC;
@ -565,7 +567,7 @@ inline simdjson_result<element> array::at(const std::string_view &json_pointer)
size_t array_index = 0;
size_t i;
for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) {
uint8_t digit = uint8_t(json_pointer[i]) - '0';
uint8_t digit = uint8_t(json_pointer[i] - '0');
// Check for non-digit in array index. If it's there, we're trying to get a field in an object
if (digit > 9) { return INCORRECT_TYPE; }
array_index = array_index*10 + digit;
@ -604,8 +606,9 @@ inline element array::iterator::operator*() const noexcept {
inline bool array::iterator::operator!=(const array::iterator& other) const noexcept {
return json_index != other.json_index;
}
inline void array::iterator::operator++() noexcept {
inline array::iterator& array::iterator::operator++() noexcept {
json_index = after_element();
return *this;
}
//
@ -703,12 +706,13 @@ inline const key_value_pair object::iterator::operator*() const noexcept {
inline bool object::iterator::operator!=(const object::iterator& other) const noexcept {
return json_index != other.json_index;
}
inline void object::iterator::operator++() noexcept {
inline object::iterator& object::iterator::operator++() noexcept {
json_index++;
json_index = after_element();
return *this;
}
inline std::string_view object::iterator::key() const noexcept {
size_t string_buf_index = tape_value();
size_t string_buf_index = size_t(tape_value());
uint32_t len;
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
return std::string_view(
@ -717,7 +721,7 @@ inline std::string_view object::iterator::key() const noexcept {
);
}
inline const char* object::iterator::key_c_str() const noexcept {
return reinterpret_cast<const char *>(&doc->string_buf[tape_value() + sizeof(uint32_t)]);
return reinterpret_cast<const char *>(&doc->string_buf[size_t(tape_value()) + sizeof(uint32_t)]);
}
inline element object::iterator::value() const noexcept {
return element(doc, json_index + 1);
@ -757,7 +761,7 @@ template<>
inline simdjson_result<const char *> element::get<const char *>() const noexcept {
switch (tape_ref_type()) {
case internal::tape_type::STRING: {
size_t string_buf_index = tape_value();
size_t string_buf_index = size_t(tape_value());
return reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]);
}
default:
@ -781,7 +785,7 @@ inline simdjson_result<uint64_t> element::get<uint64_t>() const noexcept {
if (result < 0) {
return NUMBER_OUT_OF_RANGE;
}
return static_cast<uint64_t>(result);
return uint64_t(result);
}
return INCORRECT_TYPE;
}
@ -815,9 +819,9 @@ inline simdjson_result<double> element::get<double>() const noexcept {
// information. (This could also be solved with profile-guided optimization.)
if(unlikely(!is_double())) { // branch rarely taken
if(is_uint64()) {
return next_tape_value<uint64_t>();
return double(next_tape_value<uint64_t>());
} else if(is_int64()) {
return next_tape_value<int64_t>();
return double(next_tape_value<int64_t>());
}
return INCORRECT_TYPE;
}
@ -1096,27 +1100,27 @@ really_inline tape_ref::tape_ref(const document *_doc, size_t _json_index) noexc
// most significant 8 bits.
really_inline bool tape_ref::is_double() const noexcept {
constexpr uint64_t tape_double = static_cast<uint64_t>(tape_type::DOUBLE)<<56;
constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56;
return doc->tape[json_index] == tape_double;
}
really_inline bool tape_ref::is_int64() const noexcept {
constexpr uint64_t tape_int64 = static_cast<uint64_t>(tape_type::INT64)<<56;
constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56;
return doc->tape[json_index] == tape_int64;
}
really_inline bool tape_ref::is_uint64() const noexcept {
constexpr uint64_t tape_uint64 = static_cast<uint64_t>(tape_type::UINT64)<<56;
constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56;
return doc->tape[json_index] == tape_uint64;
}
really_inline bool tape_ref::is_false() const noexcept {
constexpr uint64_t tape_false = static_cast<uint64_t>(tape_type::FALSE_VALUE)<<56;
constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56;
return doc->tape[json_index] == tape_false;
}
really_inline bool tape_ref::is_true() const noexcept {
constexpr uint64_t tape_true = static_cast<uint64_t>(tape_type::TRUE_VALUE)<<56;
constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56;
return doc->tape[json_index] == tape_true;
}
really_inline bool tape_ref::is_null_on_tape() const noexcept {
constexpr uint64_t tape_null = static_cast<uint64_t>(tape_type::NULL_VALUE)<<56;
constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56;
return doc->tape[json_index] == tape_null;
}
@ -1140,10 +1144,10 @@ really_inline uint64_t internal::tape_ref::tape_value() const noexcept {
return doc->tape[json_index] & internal::JSON_VALUE_MASK;
}
really_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept {
return static_cast<uint32_t>(doc->tape[json_index]);
return uint32_t(doc->tape[json_index]);
}
really_inline uint32_t internal::tape_ref::scope_count() const noexcept {
return static_cast<uint32_t>((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK);
return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK);
}
template<typename T>
@ -1158,7 +1162,7 @@ really_inline T tape_ref::next_tape_value() const noexcept {
return x;
}
inline std::string_view internal::tape_ref::get_string_view() const noexcept {
size_t string_buf_index = tape_value();
size_t string_buf_index = size_t(tape_value());
uint32_t len;
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
return std::string_view(

View File

@ -33,7 +33,7 @@ namespace internal {
* complete
* document, therefore the last json buffer location is the end of the batch
* */
inline size_t find_last_json_buf_idx(const uint8_t *buf, size_t size, const dom::parser &parser) {
inline uint32_t find_last_json_buf_idx(const uint8_t *buf, size_t size, const dom::parser &parser) {
// this function can be generally useful
if (parser.n_structural_indexes == 0)
return 0;
@ -85,7 +85,7 @@ static inline bool is_ascii(char c) {
return ((unsigned char)c) <= 127;
}
// if the string ends with UTF-8 values, backtrack
// if the string ends with UTF-8 values, backtrack
// up to the first ASCII character. May return 0.
static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) {
while ((len > 0) and (not is_ascii(c[len - 1]))) {
@ -100,14 +100,19 @@ static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) {
namespace simdjson {
namespace dom {
really_inline document_stream::document_stream(
dom::parser &_parser,
const uint8_t *buf,
size_t len,
size_t batch_size,
error_code _error
) noexcept : parser{_parser}, _buf{buf}, _len{len}, _batch_size(batch_size), error{_error} {
) noexcept
: parser{_parser},
_buf{buf},
_len{len},
_batch_size(batch_size),
error(_error)
{
if (!error) { error = json_parse(); }
}
@ -172,7 +177,7 @@ inline error_code document_stream::json_parse() noexcept {
if (stage1_is_ok != simdjson::SUCCESS) {
return stage1_is_ok;
}
size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
uint32_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
if (last_index == 0) {
if (parser.n_structural_indexes == 0) {
return simdjson::EMPTY;
@ -249,7 +254,7 @@ inline error_code document_stream::json_parse() noexcept {
if (stage1_is_ok != simdjson::SUCCESS) {
return stage1_is_ok;
}
size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
uint32_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
if (last_index == 0) {
if (parser.n_structural_indexes == 0) {
return EMPTY;

View File

@ -5,6 +5,10 @@
namespace simdjson {
// VS2017 reports deprecated warnings when you define a deprecated class's methods.
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_DEPRECATED_WARNING
// Because of template weirdness, the actual class definition is inline in the document class
WARN_UNUSED bool dom::parser::Iterator::is_ok() const {
@ -54,7 +58,7 @@ bool dom::parser::Iterator::move_forward() {
location += 1;
current_val = doc.tape[location];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
return true;
}
@ -62,7 +66,7 @@ void dom::parser::Iterator::move_to_value() {
// assume that we are on a key, so move by 1.
location += 1;
current_val = doc.tape[location];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
}
bool dom::parser::Iterator::move_to_key(const char *key) {
@ -139,14 +143,14 @@ bool dom::parser::Iterator::prev() {
oldnpos = npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = static_cast<uint32_t>(current_val);
npos = uint32_t(current_val);
} else {
npos = npos + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
}
} while (npos < target_location);
location = oldnpos;
current_val = doc.tape[location];
current_type = current_val >> 56;
current_type = uint8_t(current_val >> 56);
return true;
}
@ -159,7 +163,7 @@ bool dom::parser::Iterator::up() {
depth--;
location -= 1;
current_val = doc.tape[location];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
return true;
}
@ -168,7 +172,7 @@ bool dom::parser::Iterator::down() {
return false;
}
if ((current_type == '[') || (current_type == '{')) {
size_t npos = static_cast<uint32_t>(current_val);
size_t npos = uint32_t(current_val);
if (npos == location + 2) {
return false; // we have an empty scope
}
@ -178,7 +182,7 @@ bool dom::parser::Iterator::down() {
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
current_val = doc.tape[location];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
return true;
}
return false;
@ -187,19 +191,19 @@ bool dom::parser::Iterator::down() {
void dom::parser::Iterator::to_start_scope() {
location = depth_index[depth].start_of_scope;
current_val = doc.tape[location];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
}
bool dom::parser::Iterator::next() {
size_t npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = static_cast<uint32_t>(current_val);
npos = uint32_t(current_val);
} else {
npos = location + (is_number() ? 2 : 1);
}
uint64_t next_val = doc.tape[npos];
uint8_t next_type = (next_val >> 56);
uint8_t next_type = uint8_t(next_val >> 56);
if ((next_type == ']') || (next_type == '}')) {
return false; // we reached the end of the scope
}
@ -208,9 +212,9 @@ bool dom::parser::Iterator::next() {
current_type = next_type;
return true;
}
dom::parser::Iterator::Iterator(const dom::parser &pj) noexcept(false)
: doc(pj.doc), depth(0), location(0), tape_length(0) {
: doc(pj.doc)
{
#if SIMDJSON_EXCEPTIONS
if (!pj.valid) { throw simdjson_error(pj.error); }
#else
@ -221,26 +225,31 @@ dom::parser::Iterator::Iterator(const dom::parser &pj) noexcept(false)
depth_index = new scopeindex_t[max_depth + 1];
depth_index[0].start_of_scope = location;
current_val = doc.tape[location++];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
depth_index[0].scope_type = current_type;
tape_length = current_val & internal::JSON_VALUE_MASK;
tape_length = size_t(current_val & internal::JSON_VALUE_MASK);
if (location < tape_length) {
// If we make it here, then depth_capacity must >=2, but the compiler
// may not know this.
current_val = doc.tape[location];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
depth++;
assert(depth < max_depth);
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
}
}
dom::parser::Iterator::Iterator(
const dom::parser::Iterator &o) noexcept
: doc(o.doc), max_depth(o.depth), depth(o.depth), location(o.location),
tape_length(o.tape_length), current_type(o.current_type),
current_val(o.current_val) {
: doc(o.doc),
max_depth(o.depth),
depth(o.depth),
location(o.location),
tape_length(o.tape_length),
current_type(o.current_type),
current_val(o.current_val),
depth_index()
{
depth_index = new scopeindex_t[max_depth+1];
memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0]));
}
@ -287,7 +296,7 @@ bool dom::parser::Iterator::print(std::ostream &os, bool escape_strings) const {
case '}': // we end an object
case '[': // we start an array
case ']': // we end an array
os << static_cast<char>(current_type);
os << char(current_type);
break;
default:
return false;
@ -314,7 +323,7 @@ bool dom::parser::Iterator::move_to(const char *pointer,
new_pointer[new_length] = '\\';
new_length++;
}
new_pointer[new_length] = fragment;
new_pointer[new_length] = char(fragment);
i += 3;
#if __cpp_exceptions
} catch (std::invalid_argument &) {
@ -427,7 +436,7 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
bool found = false;
if (is_object()) {
if (move_to_key(key_or_index.c_str(), key_or_index.length())) {
if (move_to_key(key_or_index.c_str(), uint32_t(key_or_index.length()))) {
found = relative_move_to(pointer + offset, length - offset);
}
} else if (is_array()) {
@ -439,14 +448,14 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
size_t npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = static_cast<uint32_t>(current_val);
npos = uint32_t(current_val);
} else {
npos =
location + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
}
location = npos;
current_val = doc.tape[npos];
current_type = (current_val >> 56);
current_type = uint8_t(current_val >> 56);
return true; // how could it fail ?
}
} else { // regular numeric index
@ -469,6 +478,8 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
return found;
}
SIMDJSON_POP_DISABLE_WARNINGS
} // namespace simdjson
#endif // SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H

View File

@ -49,7 +49,7 @@ inline std::ostream& operator<<(std::ostream& out, const escape_json_string &une
if ((unsigned char)unescaped.str[i] <= 0x1F) {
// TODO can this be done once at the beginning, or will it mess up << char?
std::ios::fmtflags f(out.flags());
out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(unescaped.str[i]);
out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << int(unescaped.str[i]);
out.flags(f);
} else {
out << unescaped.str[i];

View File

@ -22,6 +22,8 @@ public:
inline Iterator(const Iterator &o) noexcept;
inline ~Iterator() noexcept;
inline Iterator& operator=(const Iterator&) = delete;
inline bool is_ok() const;
// useful for debugging purposes
@ -187,7 +189,7 @@ public:
// is referenced is undefined, and evaluation fails". Here we just return
// the first corresponding value.
inline bool move_to(const std::string &pointer) {
return move_to(pointer.c_str(), pointer.length());
return move_to(pointer.c_str(), uint32_t(pointer.length()));
}
private:
@ -253,13 +255,13 @@ public:
private:
const document &doc;
size_t max_depth;
size_t depth;
size_t location; // our current location on a tape
size_t tape_length;
uint8_t current_type;
uint64_t current_val;
scopeindex_t *depth_index;
size_t max_depth{};
size_t depth{};
size_t location{}; // our current location on a tape
size_t tape_length{};
uint8_t current_type{};
uint64_t current_val{};
scopeindex_t *depth_index{};
};
} // namespace simdjson

View File

@ -0,0 +1,3 @@
set(SIMDJSON_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
add_library(jsonchecker-data INTERFACE)
target_compile_definitions(jsonchecker-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")

View File

@ -0,0 +1,5 @@
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/twitter.json PARENT_SCOPE)
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_SOURCE_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
add_library(jsonexamples-data INTERFACE)
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")

View File

@ -0,0 +1,26 @@
#
# Amalgamation
#
# We should check whether bash is available here and avoid failures on systems where
# bash is unavailable.
if (NOT MSVC)
##
# Important! The script amalgamate.sh is not generally executable. It
# assumes that bash is at /bin/bash which may not be true.
###
set(SINGLEHEADER_FILES simdjson.h simdjson.cpp amalgamate_demo.cpp README.md)
add_custom_command(
OUTPUT ${SINGLEHEADER_FILES}
COMMAND ${CMAKE_COMMAND} -E env
AMALGAMATE_SOURCE_PATH=${PROJECT_SOURCE_DIR}/src
AMALGAMATE_INPUT_PATH=${PROJECT_SOURCE_DIR}/include
AMALGAMATE_OUTPUT_PATH=${CMAKE_CURRENT_BINARY_DIR}
bash ${CMAKE_CURRENT_SOURCE_DIR}/amalgamate.sh
DEPENDS amalgamate.sh simdjson-source
)
add_custom_target(amalgamate DEPENDS ${SINGLEHEADER_FILES})
add_executable(amalgamate_demo amalgamate_demo.cpp)
target_link_libraries(amalgamate_demo simdjson-include-source)
add_test(amalgamate_demo amalgamate_demo ${EXAMPLE_JSON} ${EXAMPLE_NDJSON})
endif()

View File

@ -1 +1,2 @@
c++ -O3 -std=c++17 -pthread -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
Try :
c++ -O3 -std=c++17 -pthread -o amalgamate_demo amalgamate_demo.cpp && ./amalgamate_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson

View File

@ -3,16 +3,16 @@
# Generates an "amalgamation build" for roaring. Inspired by similar
# script used by whefs.
########################################################################
set -e
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
echo "We are about to amalgamate all simdjson files into one source file. "
echo "See https://www.sqlite.org/amalgamation.html and https://en.wikipedia.org/wiki/Single_Compilation_Unit for rationale. "
AMAL_H="simdjson.h"
AMAL_C="simdjson.cpp"
SRCPATH="$SCRIPTPATH/src"
INCLUDEPATH="$SCRIPTPATH/include"
if [ -z "$AMALGAMATE_SOURCE_PATH" ]; then AMALGAMATE_SOURCE_PATH="$SCRIPTPATH/../src"; fi
if [ -z "$AMALGAMATE_INCLUDE_PATH" ]; then AMALGAMATE_INCLUDE_PATH="$SCRIPTPATH/../include"; fi
if [ -z "$AMALGAMATE_OUTPUT_PATH" ]; then AMALGAMATE_OUTPUT_PATH="$SCRIPTPATH"; fi
# this list excludes the "src/generic headers"
ALLCFILES="
@ -27,14 +27,14 @@ simdjson.h
found_includes=()
for file in ${ALLCFILES}; do
test -e "$SRCPATH/$file" && continue
echo "FATAL: source file [$SRCPATH/$file] not found."
test -e "$AMALGAMATE_SOURCE_PATH/$file" && continue
echo "FATAL: source file [$AMALGAMATE_SOURCE_PATH/$file] not found."
exit 127
done
for file in ${ALLCHEADERS}; do
test -e "$INCLUDEPATH/$file" && continue
echo "FATAL: source file [$INCLUDEPATH/$file] not found."
test -e "$AMALGAMATE_INCLUDE_PATH/$file" && continue
echo "FATAL: source file [$AMALGAMATE_INCLUDE_PATH/$file] not found."
exit 127
done
@ -42,18 +42,18 @@ function doinclude()
{
file=$1
line="${@:2}"
if [ -f $INCLUDEPATH/$file ]; then
if [ -f $AMALGAMATE_INCLUDE_PATH/$file ]; then
if [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
found_includes+=("$file")
dofile $INCLUDEPATH/$file
dofile $AMALGAMATE_INCLUDE_PATH $file
fi;
elif [ -f $SRCPATH/$file ]; then
elif [ -f $AMALGAMATE_SOURCE_PATH/$file ]; then
# generic includes are included multiple times
if [[ "${file}" == *'generic/'*'.h' ]]; then
dofile $SRCPATH/$file
dofile $AMALGAMATE_SOURCE_PATH $file
elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
found_includes+=("$file")
dofile $SRCPATH/$file
dofile $AMALGAMATE_SOURCE_PATH $file
else
echo "/* $file already included: $line */"
fi
@ -65,9 +65,9 @@ function doinclude()
function dofile()
{
file="$1/$2"
# Last lines are always ignored. Files should end by an empty lines.
RELFILE=${1#"$SCRIPTPATH/"}
echo "/* begin file $RELFILE */"
echo "/* begin file ${2} */"
# echo "#line 8 \"$1\"" ## redefining the line/file is not nearly as useful as it sounds for debugging. It breaks IDEs.
while IFS= read -r line || [ -n "$line" ];
do
@ -84,23 +84,31 @@ function dofile()
# Otherwise we simply copy the line
echo "$line"
fi
done < "$1"
done < "$file"
echo "/* end file $RELFILE */"
}
timestamp=$(date)
mkdir -p $AMALGAMATE_OUTPUT_PATH
AMAL_H="${AMALGAMATE_OUTPUT_PATH}/simdjson.h"
AMAL_C="${AMALGAMATE_OUTPUT_PATH}/simdjson.cpp"
DEMOCPP="${AMALGAMATE_OUTPUT_PATH}/amalgamate_demo.cpp"
README="$AMALGAMATE_OUTPUT_PATH/README.md"
echo "Creating ${AMAL_H}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_H}"
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_H}
{
for h in ${ALLCHEADERS}; do
doinclude $h "ERROR $h not found"
done
} >> "${AMAL_H}"
} >> ${AMAL_H}
echo "Creating ${AMAL_C}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_C}
{
echo "#include \"${AMAL_H}\""
echo "#include \"simdjson.h\""
echo ""
echo "/* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */"
@ -110,14 +118,13 @@ echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
echo ""
for file in ${ALLCFILES}; do
dofile "$SRCPATH/$file"
dofile $AMALGAMATE_SOURCE_PATH $file
done
} >> "${AMAL_C}"
} >> ${AMAL_C}
DEMOCPP="amalgamation_demo.cpp"
echo "Creating ${DEMOCPP}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${DEMOCPP}"
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${DEMOCPP}
cat <<< '
#include <iostream>
#include "simdjson.h"
@ -128,11 +135,14 @@ int main(int argc, char *argv[]) {
}
const char * filename = argv[1];
simdjson::dom::parser parser;
auto [doc, error] = parser.load(filename); // do the parsing
simdjson::error_code error;
UNUSED simdjson::dom::element elem;
parser.load(filename).tie(elem, error); // do the parsing
if (error) {
std::cout << "parse failed" << std::endl;
std::cout << "error code: " << error << std::endl;
std::cout << error << std::endl;
return EXIT_FAILURE;
} else {
std::cout << "parse valid" << std::endl;
}
@ -149,34 +159,31 @@ int main(int argc, char *argv[]) {
std::cout << "parse_many failed" << std::endl;
std::cout << "error code: " << error << std::endl;
std::cout << error << std::endl;
return EXIT_FAILURE;
} else {
std::cout << "parse_many valid" << std::endl;
}
return EXIT_SUCCESS;
}
' >> "${DEMOCPP}"
' >> ${DEMOCPP}
echo "Done with all files generation. "
CPPBIN=$(basename ${DEMOCPP} .cpp)
echo "Files have been written to directory: $PWD "
ls -la ${AMAL_C} ${AMAL_H} ${DEMOCPP}
echo "Try :" > ${README}
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP##*/} && ./${CPPBIN##*/} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" >> ${README}
echo "Done with all files generation."
echo "Files have been written to directory: ${AMALGAMATE_OUTPUT_PATH}/"
ls -la ${AMAL_C} ${AMAL_H} ${DEMOCPP} ${README}
#
# Instructions to create demo
#
echo ""
echo "Giving final instructions:"
CPPBIN=${DEMOCPP%%.*}
echo "Try :"
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson"
SINGLEHDR=$SCRIPTPATH/singleheader
echo "Copying files to $SCRIPTPATH/singleheader "
mkdir -p $SINGLEHDR
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md
cp ${AMAL_C} ${AMAL_H} ${DEMOCPP} $SINGLEHDR
ls $SINGLEHDR
cd $SINGLEHDR && c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
cat ${README}
lowercase(){
echo "$1" | tr 'A-Z' 'a-z'

View File

@ -1,4 +1,4 @@
/* auto-generated on Sun Apr 19 11:36:20 PDT 2020. Do not edit! */
/* auto-generated on Mon Apr 20 11:05:12 PDT 2020. Do not edit! */
#include <iostream>
#include "simdjson.h"
@ -9,11 +9,14 @@ int main(int argc, char *argv[]) {
}
const char * filename = argv[1];
simdjson::dom::parser parser;
auto [doc, error] = parser.load(filename); // do the parsing
simdjson::error_code error;
UNUSED simdjson::dom::element elem;
parser.load(filename).tie(elem, error); // do the parsing
if (error) {
std::cout << "parse failed" << std::endl;
std::cout << "error code: " << error << std::endl;
std::cout << error << std::endl;
return EXIT_FAILURE;
} else {
std::cout << "parse valid" << std::endl;
}
@ -30,6 +33,7 @@ int main(int argc, char *argv[]) {
std::cout << "parse_many failed" << std::endl;
std::cout << "error code: " << error << std::endl;
std::cout << error << std::endl;
return EXIT_FAILURE;
} else {
std::cout << "parse_many valid" << std::endl;
}

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,5 @@
/* auto-generated on Sun Apr 19 11:36:20 PDT 2020. Do not edit! */
/* begin file include/simdjson.h */
/* auto-generated on Mon Apr 20 11:05:12 PDT 2020. Do not edit! */
/* begin file simdjson.h */
#ifndef SIMDJSON_H
#define SIMDJSON_H
@ -9,7 +9,7 @@
* Check the [README.md](https://github.com/lemire/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second).
*/
/* begin file include/simdjson/compiler_check.h */
/* begin file simdjson/compiler_check.h */
#ifndef SIMDJSON_COMPILER_CHECK_H
#define SIMDJSON_COMPILER_CHECK_H
@ -45,10 +45,10 @@
#endif
#endif // SIMDJSON_COMPILER_CHECK_H
/* end file include/simdjson/compiler_check.h */
/* end file */
// Public API
/* begin file include/simdjson/simdjson_version.h */
/* begin file simdjson/simdjson_version.h */
// /include/simdjson/simdjson_version.h automatically generated by release.py,
// do not change by hand
#ifndef SIMDJSON_SIMDJSON_VERSION_H
@ -75,17 +75,17 @@ enum {
} // namespace simdjson
#endif // SIMDJSON_SIMDJSON_VERSION_H
/* end file include/simdjson/simdjson_version.h */
/* begin file include/simdjson/error.h */
/* end file */
/* begin file simdjson/error.h */
#ifndef SIMDJSON_ERROR_H
#define SIMDJSON_ERROR_H
/* begin file include/simdjson/common_defs.h */
/* begin file simdjson/common_defs.h */
#ifndef SIMDJSON_COMMON_DEFS_H
#define SIMDJSON_COMMON_DEFS_H
#include <cassert>
/* begin file include/simdjson/portability.h */
/* begin file simdjson/portability.h */
#ifndef SIMDJSON_PORTABILITY_H
#define SIMDJSON_PORTABILITY_H
@ -242,7 +242,7 @@ static inline void aligned_free_char(char *mem_block) {
}
} // namespace simdjson
#endif // SIMDJSON_PORTABILITY_H
/* end file include/simdjson/portability.h */
/* end file */
namespace simdjson {
@ -367,7 +367,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
#if (!SIMDJSON_CPLUSPLUS17)
SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
/* begin file include/simdjson/nonstd/string_view.hpp */
/* begin file simdjson/nonstd/string_view.hpp */
// Copyright 2017-2019 by Martin Moene
//
// string-view lite, a C++17-like string_view for C++98 and later.
@ -1897,7 +1897,7 @@ nssv_RESTORE_WARNINGS()
#endif // nssv_HAVE_STD_STRING_VIEW
#endif // NONSTD_SV_LITE_H_INCLUDED
/* end file include/simdjson/nonstd/string_view.hpp */
/* end file */
SIMDJSON_POP_DISABLE_WARNINGS
namespace std {
@ -1906,7 +1906,7 @@ namespace std {
#endif // if (SIMDJSON_CPLUSPLUS < 201703L)
#endif // SIMDJSON_COMMON_DEFS_H
/* end file include/simdjson/nonstd/string_view.hpp */
/* end file */
#include <string>
#include <utility>
@ -2136,8 +2136,8 @@ inline const std::string &error_message(int error) noexcept;
} // namespace simdjson
#endif // SIMDJSON_ERROR_H
/* end file include/simdjson/nonstd/string_view.hpp */
/* begin file include/simdjson/padded_string.h */
/* end file */
/* begin file simdjson/padded_string.h */
#ifndef SIMDJSON_PADDED_STRING_H
#define SIMDJSON_PADDED_STRING_H
@ -2277,8 +2277,8 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
} // namespace simdjson
#endif // SIMDJSON_PADDED_STRING_H
/* end file include/simdjson/padded_string.h */
/* begin file include/simdjson/implementation.h */
/* end file */
/* begin file simdjson/implementation.h */
#ifndef SIMDJSON_IMPLEMENTATION_H
#define SIMDJSON_IMPLEMENTATION_H
@ -2286,7 +2286,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
#include <string>
#include <atomic>
#include <vector>
/* begin file include/simdjson/document.h */
/* begin file simdjson/document.h */
#ifndef SIMDJSON_DOCUMENT_H
#define SIMDJSON_DOCUMENT_H
@ -2295,7 +2295,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
#include <string>
#include <limits>
#include <sstream>
/* begin file include/simdjson/simdjson.h */
/* begin file simdjson/simdjson.h */
/**
* @file
* @deprecated We'll be removing this file so it isn't confused with the top level simdjson.h
@ -2305,7 +2305,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
#endif // SIMDJSON_H
/* end file include/simdjson/simdjson.h */
/* end file */
namespace simdjson {
namespace dom {
@ -3634,7 +3634,7 @@ public:
} // namespace simdjson
#endif // SIMDJSON_DOCUMENT_H
/* end file include/simdjson/simdjson.h */
/* end file */
namespace simdjson {
@ -3864,8 +3864,8 @@ extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> activ
} // namespace simdjson
#endif // SIMDJSON_IMPLEMENTATION_H
/* end file include/simdjson/simdjson.h */
/* begin file include/simdjson/document_stream.h */
/* end file */
/* begin file simdjson/document_stream.h */
#ifndef SIMDJSON_DOCUMENT_STREAM_H
#define SIMDJSON_DOCUMENT_STREAM_H
@ -4012,16 +4012,16 @@ private:
} // namespace simdjson
#endif // SIMDJSON_DOCUMENT_STREAM_H
/* end file include/simdjson/document_stream.h */
/* end file */
// // Deprecated API
/* begin file include/simdjson/jsonparser.h */
/* begin file simdjson/jsonparser.h */
// TODO Remove this -- deprecated API and files
#ifndef SIMDJSON_JSONPARSER_H
#define SIMDJSON_JSONPARSER_H
/* begin file include/simdjson/parsedjson.h */
/* begin file simdjson/parsedjson.h */
// TODO Remove this -- deprecated API and files
#ifndef SIMDJSON_PARSEDJSON_H
@ -4037,8 +4037,8 @@ using ParsedJson [[deprecated("Use dom::parser instead")]] = dom::parser;
} // namespace simdjson
#endif
/* end file include/simdjson/parsedjson.h */
/* begin file include/simdjson/jsonioutil.h */
/* end file */
/* begin file simdjson/jsonioutil.h */
#ifndef SIMDJSON_JSONIOUTIL_H
#define SIMDJSON_JSONIOUTIL_H
@ -4064,7 +4064,7 @@ inline padded_string get_corpus(const char *path) {
} // namespace simdjson
#endif // SIMDJSON_JSONIOUTIL_H
/* end file include/simdjson/jsonioutil.h */
/* end file */
namespace simdjson {
@ -4174,8 +4174,8 @@ dom::parser build_parsed_json(const char *buf) noexcept = delete;
} // namespace simdjson
#endif
/* end file include/simdjson/jsonioutil.h */
/* begin file include/simdjson/parsedjson_iterator.h */
/* end file */
/* begin file simdjson/parsedjson_iterator.h */
// TODO Remove this -- deprecated API and files
#ifndef SIMDJSON_PARSEDJSON_ITERATOR_H
@ -4188,7 +4188,7 @@ dom::parser build_parsed_json(const char *buf) noexcept = delete;
#include <limits>
#include <stdexcept>
/* begin file include/simdjson/internal/jsonformatutils.h */
/* begin file simdjson/internal/jsonformatutils.h */
#ifndef SIMDJSON_INTERNAL_JSONFORMATUTILS_H
#define SIMDJSON_INTERNAL_JSONFORMATUTILS_H
@ -4254,7 +4254,7 @@ inline std::ostream& operator<<(std::ostream& out, const escape_json_string &une
} // namespace simdjson
#endif // SIMDJSON_INTERNAL_JSONFORMATUTILS_H
/* end file include/simdjson/internal/jsonformatutils.h */
/* end file */
namespace simdjson {
@ -4507,10 +4507,10 @@ public:
} // namespace simdjson
#endif
/* end file include/simdjson/internal/jsonformatutils.h */
/* end file */
// // Inline functions
/* begin file include/simdjson/inline/document.h */
/* begin file simdjson/inline/document.h */
#ifndef SIMDJSON_INLINE_DOCUMENT_H
#define SIMDJSON_INLINE_DOCUMENT_H
@ -5679,8 +5679,8 @@ inline std::string_view internal::tape_ref::get_string_view() const noexcept {
} // namespace simdjson
#endif // SIMDJSON_INLINE_DOCUMENT_H
/* end file include/simdjson/inline/document.h */
/* begin file include/simdjson/inline/document_stream.h */
/* end file */
/* begin file simdjson/inline/document_stream.h */
#ifndef SIMDJSON_INLINE_DOCUMENT_STREAM_H
#define SIMDJSON_INLINE_DOCUMENT_STREAM_H
@ -5961,8 +5961,8 @@ inline error_code document_stream::json_parse() noexcept {
} // namespace dom
} // namespace simdjson
#endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H
/* end file include/simdjson/inline/document_stream.h */
/* begin file include/simdjson/inline/error.h */
/* end file */
/* begin file simdjson/inline/error.h */
#ifndef SIMDJSON_INLINE_ERROR_H
#define SIMDJSON_INLINE_ERROR_H
@ -6101,8 +6101,8 @@ really_inline simdjson_result<T>::simdjson_result() noexcept
} // namespace simdjson
#endif // SIMDJSON_INLINE_ERROR_H
/* end file include/simdjson/inline/error.h */
/* begin file include/simdjson/inline/padded_string.h */
/* end file */
/* begin file simdjson/inline/padded_string.h */
#ifndef SIMDJSON_INLINE_PADDED_STRING_H
#define SIMDJSON_INLINE_PADDED_STRING_H
@ -6242,8 +6242,8 @@ inline simdjson_result<padded_string> padded_string::load(const std::string &fil
} // namespace simdjson
#endif // SIMDJSON_INLINE_PADDED_STRING_H
/* end file include/simdjson/inline/padded_string.h */
/* begin file include/simdjson/inline/parsedjson_iterator.h */
/* end file */
/* begin file simdjson/inline/parsedjson_iterator.h */
#ifndef SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
#define SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
@ -6717,7 +6717,7 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
} // namespace simdjson
#endif // SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
/* end file include/simdjson/inline/parsedjson_iterator.h */
/* end file */
#endif // SIMDJSON_H
/* end file include/simdjson/inline/parsedjson_iterator.h */
/* end file */

View File

@ -4,6 +4,7 @@
# target_link_libraries(my-program simdjson-include-source) gives you the header and source
# directories. It does not specify any compiler flags.
#
add_library(simdjson-include-source INTERFACE)
target_link_libraries(simdjson-include-source INTERFACE simdjson-headers)
target_include_directories(simdjson-include-source INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)

View File

@ -50,25 +50,18 @@ really_inline int count_ones(uint64_t input_num) {
really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
#ifdef _MSC_VER
// todo: this might fail under visual studio for ARM
return _addcarry_u64(0, value1, value2,
reinterpret_cast<unsigned __int64 *>(result));
*result = value1 + value2;
return *result < value1;
#else
return __builtin_uaddll_overflow(value1, value2,
(unsigned long long *)result);
#endif
}
#ifdef _MSC_VER
#pragma intrinsic(_umul128) // todo: this might fail under visual studio for ARM
#endif
really_inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
#ifdef _MSC_VER
// todo: this might fail under visual studio for ARM
uint64_t high;
*result = _umul128(value1, value2, &high);
return high;
*result = value1 * value2;
return !!__umulh(value1, value2);
#else
return __builtin_umulll_overflow(value1, value2, (unsigned long long *)result);
#endif

View File

@ -26,7 +26,7 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
memcpy(&val, chars, sizeof(uint64_t));
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
return (val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32;
return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
}
#define SWAR_NUMBER_PARSING

View File

@ -50,7 +50,7 @@ namespace simd {
typedef uint16_t bitmask_t;
typedef uint32_t bitmask2_t;
static really_inline simd8<bool> splat(bool _value) { return vmovq_n_u8(-(!!_value)); }
static really_inline simd8<bool> splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); }
really_inline simd8(const uint8x16_t _value) : base_u8<bool>(_value) {}
// False constructor
@ -159,8 +159,8 @@ namespace simd {
really_inline void compress(uint16_t mask, L * output) const {
// this particular implementation was inspired by work done by @animetosho
// we do it in two steps, first 8 bytes and then second 8 bytes
uint8_t mask1 = static_cast<uint8_t>(mask); // least significant 8 bits
uint8_t mask2 = static_cast<uint8_t>(mask >> 8); // most significant 8 bits
uint8_t mask1 = uint8_t(mask); // least significant 8 bits
uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits
// next line just loads the 64-bit values thintable_epi8[mask1] and
// thintable_epi8[mask2] into a 128-bit register, using only
// two instructions on most compilers.
@ -309,10 +309,10 @@ namespace simd {
}
really_inline void compress(uint64_t mask, T * output) const {
this->chunks[0].compress(mask, output);
this->chunks[1].compress(mask >> 16, output + 16 - count_ones(mask & 0xFFFF));
this->chunks[2].compress(mask >> 32, output + 32 - count_ones(mask & 0xFFFFFFFF));
this->chunks[3].compress(mask >> 48, output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
this->chunks[0].compress(uint16_t(mask), output);
this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF));
this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF));
this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
}
template <typename F>

View File

@ -40,8 +40,8 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8
// smash them together into a 64-byte mask and get the bitmask from there.
uint64_t bs_and_quote = simd8x64<bool>(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask();
return {
static_cast<uint32_t>(bs_and_quote), // bs_bits
static_cast<uint32_t>(bs_and_quote >> 32) // quote_bits
uint32_t(bs_and_quote), // bs_bits
uint32_t(bs_and_quote >> 32) // quote_bits
};
}

View File

@ -94,7 +94,7 @@ really_inline uint8_t *parser::on_start_string() noexcept {
}
really_inline bool parser::on_end_string(uint8_t *dst) noexcept {
uint32_t str_length = dst - (current_string_buf_loc + sizeof(uint32_t));
uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t)));
// TODO check for overflow in case someone has a crazy string (>=4GB?)
// But only add the overflow check when the document itself exceeds 4GB
// Currently unneeded because we refuse to parse docs larger or equal to 4GB.
@ -126,7 +126,7 @@ really_inline bool parser::on_number_double(double value) noexcept {
}
really_inline void parser::write_tape(uint64_t val, internal::tape_type t) noexcept {
doc.tape[current_loc++] = val | ((static_cast<uint64_t>(static_cast<char>(t))) << 56);
doc.tape[current_loc++] = val | ((uint64_t(char(t))) << 56);
}
// this function is responsible for annotating the start of the scope
@ -136,7 +136,7 @@ really_inline void parser::end_scope(uint32_t depth) noexcept {
// the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff).
const uint32_t cntsat = d.count > 0xFFFFFF ? 0xFFFFFF : d.count;
// This is a load and an OR. It would be possible to just write once at doc.tape[d.tape_index]
doc.tape[d.tape_index] |= current_loc | (static_cast<uint64_t>(cntsat) << 32);
doc.tape[d.tape_index] |= current_loc | (uint64_t(cntsat) << 32);
}
} // namespace simdjson

View File

@ -7,6 +7,23 @@
namespace simdjson {
namespace fallback {
#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64)
static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) {
unsigned long x0 = (unsigned long)x, top, bottom;
_BitScanForward(&top, (unsigned long)(x >> 32));
_BitScanForward(&bottom, x0);
*ret = x0 ? bottom : 32 + top;
return x != 0;
}
static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) {
unsigned long x1 = (unsigned long)(x >> 32), top, bottom;
_BitScanReverse(&top, x1);
_BitScanReverse(&bottom, (unsigned long)x);
*ret = x1 ? top + 32 : bottom;
return x != 0;
}
#endif
// We sometimes call trailing_zero on inputs that are zero,
// but the algorithms do not end up using the returned value.
// Sadly, sanitizers are not smart enough to figure it out.

View File

@ -127,7 +127,7 @@ really_inline error_code scan() {
}
*next_structural_index = len;
next_structural_index++;
doc_parser.n_structural_indexes = next_structural_index - doc_parser.structural_indexes.get();
doc_parser.n_structural_indexes = uint32_t(next_structural_index - doc_parser.structural_indexes.get());
return error;
}
@ -148,7 +148,7 @@ WARN_UNUSED error_code implementation::stage1(const uint8_t *buf, size_t len, pa
if (unlikely(len > parser.capacity())) {
return CAPACITY;
}
stage1::structural_scanner scanner(buf, len, parser, streaming);
stage1::structural_scanner scanner(buf, uint32_t(len), parser, streaming);
return scanner.scan();
}
@ -201,7 +201,7 @@ WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, ui
pos += meta[2] | quote;
i += 1;
nonescape = (~nonescape) | (meta[1]);
nonescape = uint8_t(~nonescape) | (meta[1]);
}
dst_len = pos; // we intentionally do not work with a reference
// for fear of aliasing

View File

@ -11,12 +11,14 @@ public:
static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept;
private:
really_inline json_minifier(uint8_t *_dst) : dst{_dst} {}
really_inline json_minifier(uint8_t *_dst)
: dst{_dst}
{}
template<size_t STEP_SIZE>
really_inline void step(const uint8_t *block_buf, buf_block_reader<STEP_SIZE> &reader) noexcept;
really_inline void next(simd::simd8x64<uint8_t> in, json_block block);
really_inline error_code finish(uint8_t *dst_start, size_t &dst_len);
json_scanner scanner;
json_scanner scanner{};
uint8_t *dst;
};
@ -70,4 +72,4 @@ error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, s
return minifier.finish(dst, dst_len);
}
} // namespace stage1
} // namespace stage1

View File

@ -48,6 +48,7 @@ private:
*/
class json_scanner {
public:
json_scanner() {}
really_inline json_block next(const simd::simd8x64<uint8_t> in);
really_inline error_code finish(bool streaming);
@ -55,7 +56,7 @@ private:
// Whether the last character of the previous iteration is part of a scalar token
// (anything except whitespace or an operator).
uint64_t prev_scalar = 0ULL;
json_string_scanner string_scanner;
json_string_scanner string_scanner{};
};
@ -74,7 +75,7 @@ really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) {
//
// Check if the current character follows a matching character, with possible "filler" between.
// For example, this checks for empty curly braces, e.g.
// For example, this checks for empty curly braces, e.g.
//
// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { <whitespace>* }
//
@ -100,4 +101,4 @@ really_inline error_code json_scanner::finish(bool streaming) {
return string_scanner.finish(streaming);
}
} // namespace stage1
} // namespace stage1

View File

@ -107,7 +107,7 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64<u
const uint64_t in_string = prefix_xor(quote) ^ prev_in_string;
// right shift of a signed value expected to be well-defined and standard
// compliant as of C++20, John Regher from Utah U. says this is fine code
prev_in_string = static_cast<uint64_t>(static_cast<int64_t>(in_string) >> 63);
prev_in_string = uint64_t(static_cast<int64_t>(in_string) >> 63);
// Use ^ to turn the beginning quote off, and the end quote on.
return {
backslash,

View File

@ -22,7 +22,7 @@ public:
// it helps tremendously.
if (bits == 0)
return;
uint32_t cnt = count_ones(bits);
int cnt = static_cast<int>(count_ones(bits));
// Do the first 8 all together
for (int i=0; i<8; i++) {
@ -42,7 +42,7 @@ public:
// branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :)
// or the start of a value ("abc" true 123) every four characters.
if (unlikely(cnt > 16)) {
uint32_t i = 16;
int i = 16;
do {
this->tail[i] = idx + trailing_zeroes(bits);
bits = clear_lowest_bit(bits);
@ -61,13 +61,14 @@ public:
static error_code index(const uint8_t *buf, size_t len, parser &parser, bool streaming) noexcept;
private:
really_inline json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {}
really_inline json_structural_indexer(uint32_t *structural_indexes)
: indexer{structural_indexes} {}
template<size_t STEP_SIZE>
really_inline void step(const uint8_t *block, buf_block_reader<STEP_SIZE> &reader) noexcept;
really_inline void next(simd::simd8x64<uint8_t> in, json_block block, size_t idx);
really_inline error_code finish(parser &parser, size_t idx, size_t len, bool streaming);
json_scanner scanner;
json_scanner scanner{};
utf8_checker checker{};
bit_indexer indexer;
uint64_t prev_structurals = 0;
@ -77,14 +78,14 @@ private:
really_inline void json_structural_indexer::next(simd::simd8x64<uint8_t> in, json_block block, size_t idx) {
uint64_t unescaped = in.lteq(0x1F);
checker.check_next_input(in);
indexer.write(idx-64, prev_structurals); // Output *last* iteration's structurals to the parser
indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser
prev_structurals = block.structural_start();
unescaped_chars_error |= block.non_quote_inside_string(unescaped);
}
really_inline error_code json_structural_indexer::finish(parser &parser, size_t idx, size_t len, bool streaming) {
// Write out the final iteration's structurals
indexer.write(idx-64, prev_structurals);
indexer.write(uint32_t(idx-64), prev_structurals);
error_code error = scanner.finish(streaming);
if (unlikely(error != SUCCESS)) { return error; }
@ -93,7 +94,7 @@ really_inline error_code json_structural_indexer::finish(parser &parser, size_t
return UNESCAPED_CHARS;
}
parser.n_structural_indexes = indexer.tail - parser.structural_indexes.get();
parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get());
/* a valid JSON file cannot have zero structural indexes - we should have
* found something */
if (unlikely(parser.n_structural_indexes == 0u)) {
@ -105,7 +106,7 @@ really_inline error_code json_structural_indexer::finish(parser &parser, size_t
if (len != parser.structural_indexes[parser.n_structural_indexes - 1]) {
/* the string might not be NULL terminated, but we add a virtual NULL
* ending character. */
parser.structural_indexes[parser.n_structural_indexes++] = len;
parser.structural_indexes[parser.n_structural_indexes++] = uint32_t(len);
}
/* make it safe to dereference one beyond this array */
parser.structural_indexes[parser.n_structural_indexes] = 0;
@ -145,7 +146,7 @@ really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_b
// they can make a lot of progress before they need that information.
// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that
// to finish: utf-8 checks and generating the output from the last iteration.
//
//
// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all
// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough
// workout.
@ -172,4 +173,4 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, parser
return indexer.finish(parser, reader.block_index(), len, streaming);
}
} // namespace stage1
} // namespace stage1

View File

@ -16,7 +16,7 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
if (-22 <= power && power <= 22 && i <= 9007199254740991) {
// convert the integer into a double. This is lossless since
// 0 <= i <= 2^53 - 1.
double d = i;
double d = double(i);
//
// The general idea is as follows.
// If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then
@ -122,7 +122,7 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
///////
uint64_t upperbit = upper >> 63;
uint64_t mantissa = upper >> (upperbit + 9);
lz += 1 ^ upperbit;
lz += int(1 ^ upperbit);
// Here we have mantissa < (1<<54).
@ -276,13 +276,13 @@ never_inline bool parse_large_integer(const uint8_t *const src,
++p;
i = 0;
} else {
unsigned char digit = *p - '0';
unsigned char digit = static_cast<unsigned char>(*p - '0');
i = digit;
p++;
// the is_made_of_eight_digits_fast routine is unlikely to help here because
// we rarely see large integer parts like 123456789
while (is_integer(*p)) {
digit = *p - '0';
digit = static_cast<unsigned char>(*p - '0');
if (mul_overflow(i, 10, &i)) {
#ifdef JSON_TEST_NUMBERS // for unit testing
found_invalid_number(src);
@ -404,13 +404,13 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
#endif
return false;
}
unsigned char digit = *p - '0';
unsigned char digit = static_cast<unsigned char>(*p - '0');
i = digit;
p++;
// the is_made_of_eight_digits_fast routine is unlikely to help here because
// we rarely see large integer parts like 123456789
while (is_integer(*p)) {
digit = *p - '0';
digit = static_cast<unsigned char>(*p - '0');
// a multiplication by 10 is cheaper than an arbitrary integer
// multiplication
i = 10 * i + digit; // might overflow, we will handle the overflow later
@ -428,7 +428,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
++p;
const char *const first_after_period = p;
if (is_integer(*p)) {
unsigned char digit = *p - '0';
unsigned char digit = static_cast<unsigned char>(*p - '0');
++p;
i = i * 10 + digit; // might overflow + multiplication by 10 is likely
// cheaper than arbitrary mult.
@ -448,7 +448,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
}
#endif
while (is_integer(*p)) {
unsigned char digit = *p - '0';
unsigned char digit = static_cast<unsigned char>(*p - '0');
++p;
i = i * 10 + digit; // in rare cases, this will overflow, but that's ok
// because we have parse_highprecision_float later.
@ -456,7 +456,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
exponent = first_after_period - p;
}
int digit_count =
p - start_digits - 1; // used later to guard against overflows
int(p - start_digits) - 1; // used later to guard against overflows
int64_t exp_number = 0; // exponential part
if (('e' == *p) || ('E' == *p)) {
is_float = true;
@ -474,16 +474,16 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
#endif
return false;
}
unsigned char digit = *p - '0';
unsigned char digit = static_cast<unsigned char>(*p - '0');
exp_number = digit;
p++;
if (is_integer(*p)) {
digit = *p - '0';
digit = static_cast<unsigned char>(*p - '0');
exp_number = 10 * exp_number + digit;
++p;
}
if (is_integer(*p)) {
digit = *p - '0';
digit = static_cast<unsigned char>(*p - '0');
exp_number = 10 * exp_number + digit;
++p;
}
@ -495,7 +495,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
#endif
return false;
}
digit = *p - '0';
digit = static_cast<unsigned char>(*p - '0');
exp_number = 10 * exp_number + digit;
++p;
}
@ -513,7 +513,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
start++;
}
// we over-decrement by one when there is a '.'
digit_count -= (start - start_digits);
digit_count -= int(start - start_digits);
if (digit_count >= 19) {
// Ok, chances are good that we had an overflow!
// this is almost never going to get called!!!

View File

@ -50,7 +50,11 @@ struct unified_machine_addresses {
class structural_iterator {
public:
really_inline structural_iterator(const uint8_t* _buf, size_t _len, const uint32_t *_structural_indexes, size_t next_structural_index)
: buf{_buf}, len{_len}, structural_indexes{_structural_indexes}, next_structural{next_structural_index} {}
: buf{_buf},
len{_len},
structural_indexes{_structural_indexes},
next_structural{next_structural_index}
{}
really_inline char advance_char() {
idx = structural_indexes[next_structural];
next_structural++;
@ -105,8 +109,8 @@ public:
const size_t len;
const uint32_t* const structural_indexes;
size_t next_structural; // next structural index
size_t idx; // location of the structural character in the input (buf)
uint8_t c; // used to track the (structural) character we are looking at
size_t idx{0}; // location of the structural character in the input (buf)
uint8_t c{0}; // used to track the (structural) character we are looking at
};
struct structural_parser {

View File

@ -1,7 +1,7 @@
namespace stage2 {
struct streaming_structural_parser: structural_parser {
really_inline streaming_structural_parser(const uint8_t *_buf, size_t _len, parser &_doc_parser, size_t _i) : structural_parser(_buf, _len, _doc_parser, _i) {}
really_inline streaming_structural_parser(const uint8_t *buf, size_t len, parser &_doc_parser, uint32_t next_structural) : structural_parser(buf, len, _doc_parser, next_structural) {}
// override to add streaming
WARN_UNUSED really_inline error_code start(UNUSED size_t len, ret_address finish_parser) {
@ -41,7 +41,7 @@ struct streaming_structural_parser: structural_parser {
***********/
WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, parser &doc_parser, size_t &next_json) const noexcept {
static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES();
stage2::streaming_structural_parser parser(buf, len, doc_parser, next_json);
stage2::streaming_structural_parser parser(buf, len, doc_parser, uint32_t(next_json));
error_code result = parser.start(len, addresses.finish);
if (result) { return result; }
//

View File

@ -33,17 +33,19 @@ really_inline uint64_t clear_lowest_bit(uint64_t input_num) {
/* result might be undefined when input_num is zero */
really_inline int leading_zeroes(uint64_t input_num) {
return static_cast<int>(_lzcnt_u64(input_num));
return int(_lzcnt_u64(input_num));
}
really_inline int count_ones(uint64_t input_num) {
#ifdef _MSC_VER
really_inline unsigned __int64 count_ones(uint64_t input_num) {
// note: we do not support legacy 32-bit Windows
return __popcnt64(input_num);// Visual Studio wants two underscores
#else
return _popcnt64(input_num);
#endif
}
#else
really_inline long long int count_ones(uint64_t input_num) {
return _popcnt64(input_num);
}
#endif
really_inline bool add_overflow(uint64_t value1, uint64_t value2,
uint64_t *result) {

View File

@ -61,7 +61,7 @@ namespace simd {
// SIMD byte mask type (returned by things like eq and gt)
template<>
struct simd8<bool>: base8<bool> {
static really_inline simd8<bool> splat(bool _value) { return _mm256_set1_epi8(-(!!_value)); }
static really_inline simd8<bool> splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); }
really_inline simd8<bool>() : base8() {}
really_inline simd8<bool>(const __m256i _value) : base8<bool>(_value) {}
@ -125,10 +125,10 @@ namespace simd {
really_inline void compress(uint32_t mask, L * output) const {
// this particular implementation was inspired by work done by @animetosho
// we do it in four steps, first 8 bytes and then second 8 bytes...
uint8_t mask1 = static_cast<uint8_t>(mask); // least significant 8 bits
uint8_t mask2 = static_cast<uint8_t>(mask >> 8); // second least significant 8 bits
uint8_t mask3 = static_cast<uint8_t>(mask >> 16); // ...
uint8_t mask4 = static_cast<uint8_t>(mask >> 24); // ...
uint8_t mask1 = uint8_t(mask); // least significant 8 bits
uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits
uint8_t mask3 = uint8_t(mask >> 16); // ...
uint8_t mask4 = uint8_t(mask >> 24); // ...
// next line just loads the 64-bit values thintable_epi8[mask1] and
// thintable_epi8[mask2] into a 128-bit register, using only
// two instructions on most compilers.
@ -305,8 +305,8 @@ namespace simd {
}
really_inline void compress(uint64_t mask, T * output) const {
uint32_t mask1 = static_cast<uint32_t>(mask);
uint32_t mask2 = static_cast<uint32_t>(mask >> 32);
uint32_t mask1 = uint32_t(mask);
uint32_t mask2 = uint32_t(mask >> 32);
this->chunks[0].compress(mask1, output);
this->chunks[1].compress(mask2, output + 32 - count_ones(mask1));
}
@ -347,7 +347,7 @@ namespace simd {
}
really_inline uint64_t to_bitmask() const {
uint64_t r_lo = static_cast<uint32_t>(this->chunks[0].to_bitmask());
uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask());
uint64_t r_hi = this->chunks[1].to_bitmask();
return r_lo | (r_hi << 32);
}

View File

@ -280,27 +280,27 @@ static inline bool is_utf8_continuing(char c) {
//
inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
if (cp <= 0x7F) {
c[0] = cp;
c[0] = uint8_t(cp);
return 1; // ascii
}
if (cp <= 0x7FF) {
c[0] = (cp >> 6) + 192;
c[1] = (cp & 63) + 128;
c[0] = uint8_t((cp >> 6) + 192);
c[1] = uint8_t((cp & 63) + 128);
return 2; // universal plane
// Surrogates are treated elsewhere...
//} //else if (0xd800 <= cp && cp <= 0xdfff) {
// return 0; // surrogates // could put assert here
} else if (cp <= 0xFFFF) {
c[0] = (cp >> 12) + 224;
c[1] = ((cp >> 6) & 63) + 128;
c[2] = (cp & 63) + 128;
c[0] = uint8_t((cp >> 12) + 224);
c[1] = uint8_t(((cp >> 6) & 63) + 128);
c[2] = uint8_t((cp & 63) + 128);
return 3;
} else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this
// is not needed
c[0] = (cp >> 18) + 240;
c[1] = ((cp >> 12) & 63) + 128;
c[2] = ((cp >> 6) & 63) + 128;
c[3] = (cp & 63) + 128;
c[0] = uint8_t((cp >> 18) + 240);
c[1] = uint8_t(((cp >> 12) & 63) + 128);
c[2] = uint8_t(((cp >> 6) & 63) + 128);
c[3] = uint8_t((cp & 63) + 128);
return 4;
}
// will return 0 when the code point was too large.
@ -324,15 +324,31 @@ struct value128 {
uint64_t high;
};
#if defined(_MSC_VER) && !defined(_M_X64) // _umul128 for x86, arm, arm64
#if defined(_M_ARM)
static inline uint64_t __emulu(uint32_t x, uint32_t y) {
return x * (uint64_t)y;
}
#endif
static inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) {
uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd);
uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd);
uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32));
uint64_t adbc_carry = !!(adbc < ad);
uint64_t lo = bd + (adbc << 32);
*hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + (adbc_carry << 32) + !!(lo < bd);
return lo;
}
#endif
really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) {
value128 answer;
#ifdef _MSC_VER
// todo: this might fail under visual studio for ARM
answer.low = _umul128(value1, value2, &answer.high);
#else
__uint128_t r = ((__uint128_t)value1) * value2;
answer.low = r;
answer.high = r >> 64;
answer.low = uint64_t(r);
answer.high = uint64_t(r >> 64);
#endif
return answer;
}

View File

@ -44,14 +44,16 @@ really_inline int leading_zeroes(uint64_t input_num) {
#endif// _MSC_VER
}
really_inline int count_ones(uint64_t input_num) {
#ifdef _MSC_VER
really_inline unsigned __int64 count_ones(uint64_t input_num) {
// note: we do not support legacy 32-bit Windows
return __popcnt64(input_num);// Visual Studio wants two underscores
#else
return _popcnt64(input_num);
#endif
}
#else
really_inline long long int count_ones(uint64_t input_num) {
return _popcnt64(input_num);
}
#endif
really_inline bool add_overflow(uint64_t value1, uint64_t value2,
uint64_t *result) {

View File

@ -62,7 +62,7 @@ namespace simd {
// SIMD byte mask type (returned by things like eq and gt)
template<>
struct simd8<bool>: base8<bool> {
static really_inline simd8<bool> splat(bool _value) { return _mm_set1_epi8(-(!!_value)); }
static really_inline simd8<bool> splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); }
really_inline simd8<bool>() : base8() {}
really_inline simd8<bool>(const __m128i _value) : base8<bool>(_value) {}
@ -124,8 +124,8 @@ namespace simd {
really_inline void compress(uint16_t mask, L * output) const {
// this particular implementation was inspired by work done by @animetosho
// we do it in two steps, first 8 bytes and then second 8 bytes
uint8_t mask1 = static_cast<uint8_t>(mask); // least significant 8 bits
uint8_t mask2 = static_cast<uint8_t>(mask >> 8); // most significant 8 bits
uint8_t mask1 = uint8_t(mask); // least significant 8 bits
uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits
// next line just loads the 64-bit values thintable_epi8[mask1] and
// thintable_epi8[mask2] into a 128-bit register, using only
// two instructions on most compilers.
@ -278,10 +278,10 @@ namespace simd {
}
really_inline void compress(uint64_t mask, T * output) const {
this->chunks[0].compress(mask, output);
this->chunks[1].compress(mask >> 16, output + 16 - count_ones(mask & 0xFFFF));
this->chunks[2].compress(mask >> 32, output + 32 - count_ones(mask & 0xFFFFFFFF));
this->chunks[3].compress(mask >> 48, output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
this->chunks[0].compress(uint16_t(mask), output);
this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF));
this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF));
this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
}
template <typename F>
@ -330,7 +330,7 @@ namespace simd {
}
really_inline uint64_t to_bitmask() const {
uint64_t r0 = static_cast<uint32_t>(this->chunks[0].to_bitmask());
uint64_t r0 = uint32_t(this->chunks[0].to_bitmask());
uint64_t r1 = this->chunks[1].to_bitmask();
uint64_t r2 = this->chunks[2].to_bitmask();
uint64_t r3 = this->chunks[3].to_bitmask();

View File

@ -38,8 +38,8 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8
v1.store(dst + 16);
uint64_t bs_and_quote = simd8x64<bool>(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask();
return {
static_cast<uint32_t>(bs_and_quote), // bs_bits
static_cast<uint32_t>(bs_and_quote >> 32) // quote_bits
uint32_t(bs_and_quote), // bs_bits
uint32_t(bs_and_quote >> 32) // quote_bits
};
}

View File

@ -66,7 +66,7 @@ if (NOT MSVC) # Can't run .sh on windows
#
add_test(
NAME testjson2json
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh
COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh
WORKING_DIRECTORY $<TARGET_FILE_DIR:minify>
)
set_property(TEST testjson2json APPEND PROPERTY DEPENDS minify json2json)
@ -79,12 +79,13 @@ if (NOT MSVC) # Can't run .sh on windows
add_executable(allparserscheckfile allparserscheckfile.cpp)
target_link_libraries(allparserscheckfile competition-all)
add_test(issue150 ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
add_test(issue150 bash ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
set_property(TEST issue150 APPEND PROPERTY DEPENDS allparserscheckfile)
set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation)
endif()
endif()
# This should really test whether bash is available.
if (NOT MSVC)
#
# json2json tool test: check that json2json can parse twitter.json
@ -139,7 +140,6 @@ if(NOT (MSVC AND MSVC_VERSION LESS 1920))
# Compile tests that *should fail*
add_cpp_test(readme_examples_will_fail_with_exceptions_off WILL_FAIL COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0)
endif()

View File

@ -138,7 +138,7 @@ int main(int argc, char *argv[]) {
jsmn_init(&jsmnparser);
memcpy(buffer, p.data(), p.size());
buffer[p.size()] = '\0';
int r = jsmn_parse(&jsmnparser, buffer, p.size(), tokens.get(), p.size());
int r = jsmn_parse(&jsmnparser, buffer, p.size(), tokens.get(), static_cast<unsigned int>(p.size()));
tokens = nullptr;
jsmn_correct = (r > 0);
}

View File

@ -73,7 +73,7 @@ namespace number_tests {
std::cout << __func__ << std::endl;
char buf[1024];
simdjson::dom::parser parser;
int maxulp = 0;
uint64_t maxulp = 0;
for (int i = -1075; i < 1024; ++i) {// large negative values should be zero.
double expected = pow(2, i);
auto n = sprintf(buf, "%.*e", std::numeric_limits<double>::max_digits10 - 1, expected);
@ -81,7 +81,7 @@ namespace number_tests {
fflush(NULL);
auto [actual, error] = parser.parse(buf, n).get<double>();
if (error) { std::cerr << error << std::endl; return false; }
int ulp = f64_ulp_dist(actual,expected);
uint64_t ulp = f64_ulp_dist(actual,expected);
if(ulp > maxulp) maxulp = ulp;
if(ulp > 0) {
std::cerr << "JSON '" << buf << " parsed to " << actual << " instead of " << expected << std::endl;
@ -284,12 +284,12 @@ namespace document_tests {
data.emplace_back(std::string(buf, n));
}
for (size_t i = 0; i < n_records; ++i) {
auto n = sprintf(buf, "{\"counter\": %f, \"array\": [%s]}", i * 3.1416,
auto n = sprintf(buf, "{\"counter\": %f, \"array\": [%s]}", static_cast<double>(i) * 3.1416,
(i % 2) ? "true" : "false");
data.emplace_back(std::string(buf, n));
}
for (size_t i = 0; i < n_records; ++i) {
auto n = sprintf(buf, "{\"number\": %e}", i * 10000.31321321);
auto n = sprintf(buf, "{\"number\": %e}", static_cast<double>(i) * 10000.31321321);
data.emplace_back(std::string(buf, n));
}
data.emplace_back(std::string("true"));
@ -1563,7 +1563,7 @@ namespace type_tests {
&& (expected_value >= 0 ?
test_cast<uint64_t>(result, expected_value) :
test_cast<uint64_t>(result, NUMBER_OUT_OF_RANGE))
&& test_cast<double>(result, expected_value)
&& test_cast<double>(result, static_cast<double>(expected_value))
&& test_cast<bool>(result, INCORRECT_TYPE)
&& test_is_null(result, false);
}
@ -1582,7 +1582,7 @@ namespace type_tests {
&& test_cast<const char *>(result, INCORRECT_TYPE)
&& test_cast<int64_t>(result, NUMBER_OUT_OF_RANGE)
&& test_cast<uint64_t>(result, expected_value)
&& test_cast<double>(result, expected_value)
&& test_cast<double>(result, static_cast<double>(expected_value))
&& test_cast<bool>(result, INCORRECT_TYPE)
&& test_is_null(result, false);
}

View File

@ -31,11 +31,11 @@ static inline bool read_hex(const char *p, unsigned &u) {
while (i--) {
unsigned char c = *p++;
if (c >= '0' && c <= '9') {
c -= '0';
c = static_cast<unsigned char>(c - '0');
} else if (c >= 'a' && c <= 'f') {
c = c - 'a' + 10;
c = static_cast<unsigned char>(c - 'a' + 10);
} else if (c >= 'A' && c <= 'F') {
c = c - 'A' + 10;
c = static_cast<unsigned char>(c - 'A' + 10);
} else {
return false;
}
@ -48,20 +48,20 @@ static inline bool read_hex(const char *p, unsigned &u) {
static inline void write_utf8(unsigned codepoint, char *&end) {
if (codepoint < 0x80) {
*end++ = codepoint;
*end++ = static_cast<char>(codepoint);
} else if (codepoint < 0x800) {
*end++ = 0xC0 | (codepoint >> 6);
*end++ = 0x80 | (codepoint & 0x3F);
*end++ = static_cast<char>(0xC0 | (codepoint >> 6));
*end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
} else if (codepoint < 0x10000) {
*end++ = 0xE0 | (codepoint >> 12);
*end++ = 0x80 | ((codepoint >> 6) & 0x3F);
*end++ = 0x80 | (codepoint & 0x3F);
*end++ = static_cast<char>(0xE0 | (codepoint >> 12));
*end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
*end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
} else {
assert(codepoint < 0x200000);
*end++ = 0xF0 | (codepoint >> 18);
*end++ = 0x80 | ((codepoint >> 12) & 0x3F);
*end++ = 0x80 | ((codepoint >> 6) & 0x3F);
*end++ = 0x80 | (codepoint & 0x3F);
*end++ = static_cast<char>(0xF0 | (codepoint >> 18));
*end++ = static_cast<char>(0x80 | ((codepoint >> 12) & 0x3F));
*end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
*end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
}
}
@ -357,7 +357,7 @@ bool validate(const char *dirname) {
"\taverage string length: %.1f \n",
name, isok ? " is valid " : " is not valid ", bad_string,
good_string, empty_string,
(double)total_string_length / good_string);
static_cast<double>(total_string_length) / static_cast<double>(good_string));
} else if (bad_string > 0) {
printf("File %40s %s --- bad strings: %10zu \n", name,
isok ? " is valid " : " is not valid ", bad_string);

View File

@ -159,5 +159,5 @@ print("modified "+doxyfile+", a backup was made")
scriptlocation = os.path.dirname(os.path.abspath(__file__))
print("Please run the tests before issuing a release, do make test && make amalgamate \n")
print("Please run the tests before issuing a release, do make test && make amalgamate_test \n")
print("to issue release, enter \n git commit -a && git push && git tag -a v"+toversionstring(*newversion)+" -m \"version "+toversionstring(*newversion)+"\" && git push --tags \n")