Merge branch 'master' into jkeiser/cmake-fuzz-noexceptions
This commit is contained in:
commit
7a2fda891c
|
@ -12,14 +12,19 @@ environment:
|
|||
matrix:
|
||||
- SIMDJSON_BUILD_STATIC: ON
|
||||
SIMDJSON_ENABLE_THREADS: OFF
|
||||
SIMDJSON_PLATFORM: x64
|
||||
- SIMDJSON_BUILD_STATIC: ON
|
||||
SIMDJSON_ENABLE_THREADS: OFF
|
||||
SIMDJSON_PLATFORM: Win32
|
||||
- SIMDJSON_BUILD_STATIC: OFF
|
||||
SIMDJSON_ENABLE_THREADS: ON
|
||||
SIMDJSON_PLATFORM: x64
|
||||
|
||||
build_script:
|
||||
- set
|
||||
- mkdir build
|
||||
- cd build
|
||||
- cmake -DSIMDJSON_BUILD_STATIC=%SIMDJSON_BUILD_STATIC% -DSIMDJSON_ENABLE_THREADS=%SIMDJSON_ENABLE_THREADS% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_GENERATOR_PLATFORM=x64 ..
|
||||
- cmake -DSIMDJSON_BUILD_STATIC=%SIMDJSON_BUILD_STATIC% -DSIMDJSON_ENABLE_THREADS=%SIMDJSON_ENABLE_THREADS% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_GENERATOR_PLATFORM=%SIMDJSON_PLATFORM% ..
|
||||
- cmake -LH ..
|
||||
- cmake --build . --config %Configuration% --verbose
|
||||
|
||||
|
@ -29,7 +34,7 @@ test_script:
|
|||
matrix:
|
||||
fast_finish: true
|
||||
exclude:
|
||||
# Don't build all variants on 2019, just running it to make sure readme_tests succeed
|
||||
- image: Visual Studio 2019
|
||||
# Don't build all variants on 2017, just running it to make sure readme_tests succeed
|
||||
- image: Visual Studio 2017
|
||||
SIMDJSON_BUILD_STATIC: ON
|
||||
SIMDJSON_ENABLE_THREADS: OFF
|
||||
|
|
|
@ -25,6 +25,13 @@ executors:
|
|||
CXX: g++
|
||||
BUILD_FLAGS: -j
|
||||
CTEST_FLAGS: -j4 --output-on-failure
|
||||
clang9:
|
||||
docker:
|
||||
- image: conanio/clang9
|
||||
environment:
|
||||
CXX: clang++-9
|
||||
BUILD_FLAGS: -j
|
||||
CTEST_FLAGS: -j4 --output-on-failure
|
||||
|
||||
clang6:
|
||||
docker:
|
||||
|
@ -50,10 +57,13 @@ commands:
|
|||
- run: make test
|
||||
- run: make checkperf
|
||||
|
||||
cmake_prep:
|
||||
install_cmake:
|
||||
steps:
|
||||
- run: apt-get update -qq
|
||||
- run: apt-get install -y cmake
|
||||
|
||||
cmake_prep:
|
||||
steps:
|
||||
- checkout
|
||||
|
||||
cmake_build:
|
||||
|
@ -78,6 +88,7 @@ commands:
|
|||
- run: SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
|
||||
- run: ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now.
|
||||
|
||||
|
||||
# we not only want cmake to build and run tests, but we want also a successful installation from which we can build, link and run programs
|
||||
cmake_install_test: # this version builds, install, test and then verify from the installation
|
||||
steps:
|
||||
|
@ -91,60 +102,65 @@ jobs:
|
|||
description: Build and run tests on GCC 7 and AVX 2 with a cmake static build
|
||||
executor: gcc7
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
|
||||
steps: [ cmake_test_all, cmake_install_test ]
|
||||
steps: [ install_cmake, cmake_test_all, cmake_install_test ]
|
||||
clang6:
|
||||
description: Build and run tests on clang 6 and AVX 2 with a cmake static build
|
||||
executor: clang6
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON }
|
||||
steps: [ init_clang6, cmake_test_all, cmake_install_test ]
|
||||
|
||||
steps: [ init_clang6, install_cmake, cmake_test_all, cmake_install_test ]
|
||||
# libcpp
|
||||
libcpp-clang9:
|
||||
description: Build and run tests on clang 6 and AVX 2 with a cmake static build and libc++
|
||||
executor: clang9
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON }
|
||||
steps: [ cmake_test_all, cmake_install_test ]
|
||||
# sanitize
|
||||
sanitize-gcc9:
|
||||
description: Build and run tests on GCC 9 and AVX 2 with a cmake sanitize build
|
||||
executor: gcc9
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
|
||||
steps: [ cmake_test_all ]
|
||||
steps: [ install_cmake, cmake_test_all ]
|
||||
sanitize-clang6:
|
||||
description: Build and run tests on clang 6 and AVX 2 with a cmake sanitize build
|
||||
executor: clang6
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: -j4 --output-on-failure -E checkperf }
|
||||
steps: [ init_clang6, cmake_test_all ]
|
||||
steps: [ init_clang6, install_cmake, cmake_test_all ]
|
||||
|
||||
# dynamic
|
||||
dynamic-gcc7:
|
||||
description: Build and run tests on GCC 7 and AVX 2 with a cmake dynamic build
|
||||
executor: gcc7
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
|
||||
steps: [ cmake_test, cmake_install_test ]
|
||||
steps: [ install_cmake, cmake_test, cmake_install_test ]
|
||||
dynamic-clang6:
|
||||
description: Build and run tests on clang 6 and AVX 2 with a cmake dynamic build
|
||||
executor: clang6
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
|
||||
steps: [ init_clang6, cmake_test, cmake_install_test ]
|
||||
steps: [ init_clang6, install_cmake, cmake_test, cmake_install_test ]
|
||||
|
||||
# unthreaded
|
||||
unthreaded-gcc7:
|
||||
description: Build and run tests on GCC 7 and AVX 2 *without* threads
|
||||
executor: gcc7
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
|
||||
steps: [ cmake_test, cmake_install_test ]
|
||||
steps: [ install_cmake, cmake_test, cmake_install_test ]
|
||||
unthreaded-clang6:
|
||||
description: Build and run tests on Clang 6 and AVX 2 *without* threads
|
||||
executor: clang6
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_ENABLE_THREADS=OFF }
|
||||
steps: [ init_clang6, cmake_test, cmake_install_test ]
|
||||
steps: [ init_clang6, install_cmake, cmake_test, cmake_install_test ]
|
||||
|
||||
# noexcept
|
||||
noexcept-gcc7:
|
||||
description: Build and run tests on GCC 7 and AVX 2 with exceptions off
|
||||
executor: gcc7
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
|
||||
steps: [ cmake_test, cmake_install_test ]
|
||||
steps: [ install_cmake, cmake_test, cmake_install_test ]
|
||||
noexcept-clang6:
|
||||
description: Build and run tests on GCC 7 and AVX 2 with exceptions off
|
||||
executor: clang6
|
||||
environment: { CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF }
|
||||
steps: [ init_clang6, cmake_test, cmake_install_test ]
|
||||
steps: [ init_clang6, install_cmake, cmake_test, cmake_install_test ]
|
||||
|
||||
#
|
||||
# Misc.
|
||||
|
@ -174,7 +190,10 @@ workflows:
|
|||
# full multi-implementation tests
|
||||
- gcc7
|
||||
- clang6
|
||||
|
||||
|
||||
# libc++
|
||||
- libcpp-clang9
|
||||
|
||||
# full single-implementation tests
|
||||
- sanitize-gcc9
|
||||
- sanitize-clang6
|
||||
|
|
|
@ -0,0 +1,21 @@
|
|||
task:
|
||||
timeout_in: 120m
|
||||
freebsd_instance:
|
||||
matrix:
|
||||
- image_family: freebsd-13-0-snap
|
||||
|
||||
env:
|
||||
ASSUME_ALWAYS_YES: YES
|
||||
setup_script:
|
||||
- pkg update -f
|
||||
- pkg install bash
|
||||
- pkg install cmake
|
||||
- pkg install git
|
||||
build_script:
|
||||
- mkdir build
|
||||
- cd build
|
||||
- cmake ..
|
||||
- make -j2
|
||||
test_script:
|
||||
- cd build
|
||||
- ctest -j2 --output-on-failure -E checkperf
|
|
@ -1,7 +1,7 @@
|
|||
*
|
||||
!.git
|
||||
!Makefile
|
||||
!amalgamation.sh
|
||||
!amalgamate.sh
|
||||
!benchmark
|
||||
!dependencies
|
||||
!include
|
||||
|
|
|
@ -47,7 +47,7 @@ platform:
|
|||
steps:
|
||||
- name: build
|
||||
image: gcc:8
|
||||
commands: [ make, make amalgamate ]
|
||||
commands: [ make, make amalgamate_test ]
|
||||
---
|
||||
kind: pipeline
|
||||
name: x64-slowtests
|
||||
|
@ -87,7 +87,7 @@ steps:
|
|||
image: gcc:8
|
||||
environment:
|
||||
EXTRA_FLAGS: -fno-exceptions
|
||||
commands: [ make, make amalgamate ]
|
||||
commands: [ make, make amalgamate_test ]
|
||||
---
|
||||
kind: pipeline
|
||||
name: x64-noexceptions-slowtests
|
||||
|
@ -139,7 +139,7 @@ platform:
|
|||
steps:
|
||||
- name: build
|
||||
image: gcc:8
|
||||
commands: [ make, make amalgamate ]
|
||||
commands: [ make, make amalgamate_test ]
|
||||
---
|
||||
kind: pipeline
|
||||
name: arm64-slowtests
|
||||
|
|
|
@ -133,6 +133,11 @@ objs
|
|||
/tests/allparserscheckfile
|
||||
/tests/basictests
|
||||
/tests/checkimplementation
|
||||
/tests/compilation_failure_tests/dangling_parser_load_should_compile
|
||||
/tests/compilation_failure_tests/dangling_parser_parse_padstring_should_compile
|
||||
/tests/compilation_failure_tests/dangling_parser_parse_stdstring_should_compile
|
||||
/tests/compilation_failure_tests/dangling_parser_parse_uchar_should_compile
|
||||
/tests/compilation_failure_tests/dangling_parser_parse_uint8_should_compile
|
||||
/tests/compilation_failure_tests/example_compiletest_should_compile
|
||||
/tests/errortests
|
||||
/tests/extracting_values_example
|
||||
|
@ -213,3 +218,7 @@ _deps
|
|||
# We check in a custom version of root Makefile that is not generated by CMake
|
||||
!/Makefile
|
||||
|
||||
singleheader/amalgamation_demo
|
||||
singleheader/amalgamation_demo.cpp
|
||||
singleheader/simdjson.cpp
|
||||
singleheader/simdjson.h
|
|
@ -25,11 +25,13 @@ if(MSVC)
|
|||
else()
|
||||
option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
|
||||
option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON)
|
||||
option(SIMDJSON_USE_LIBCPP "Use the libc++ library" OFF)
|
||||
endif()
|
||||
option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" OFF)
|
||||
|
||||
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake")
|
||||
|
||||
|
||||
# We compile tools, tests, etc. with C++ 17. Override yourself if you need on a target.
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
@ -38,6 +40,7 @@ set(CMAKE_MACOSX_RPATH OFF)
|
|||
set(CMAKE_THREAD_PREFER_PTHREAD ON)
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
|
||||
|
||||
# LTO seems to create all sorts of fun problems. Let us
|
||||
# disable temporarily.
|
||||
#include(CheckIPOSupported)
|
||||
|
@ -52,12 +55,23 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
|
|||
add_library(simdjson-flags INTERFACE)
|
||||
if(MSVC)
|
||||
target_compile_options(simdjson-flags INTERFACE /nologo /D_CRT_SECURE_NO_WARNINGS)
|
||||
target_compile_options(simdjson-flags INTERFACE /WX /W3 /wd4005 /wd4996 /wd4267 /wd4244 /wd4113)
|
||||
target_compile_options(simdjson-flags INTERFACE /WX /W3)
|
||||
else()
|
||||
target_compile_options(simdjson-flags INTERFACE -fPIC)
|
||||
target_compile_options(simdjson-flags INTERFACE -Werror -Wall -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self)
|
||||
target_compile_options(simdjson-flags INTERFACE -Werror -Wall -Wextra -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self -Wconversion -Wno-sign-conversion)
|
||||
endif()
|
||||
|
||||
|
||||
if(SIMDJSON_USE_LIBCPP)
|
||||
target_link_libraries(simdjson-flags INTERFACE -stdlib=libc++ -lc++abi)
|
||||
# instead of the above line, we could have used
|
||||
# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi")
|
||||
# The next line is needed empirically.
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
|
||||
# we update CMAKE_SHARED_LINKER_FLAGS, this gets updated later as well
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -lc++abi")
|
||||
endif(SIMDJSON_USE_LIBCPP)
|
||||
|
||||
# Optional flags
|
||||
option(SIMDJSON_IMPLEMENTATION_HASWELL "Include the haswell implementation" ON)
|
||||
if(NOT SIMDJSON_IMPLEMENTATION_HASWELL)
|
||||
|
@ -138,6 +152,15 @@ if (NOT MSVC)
|
|||
add_custom_target(simdjson-user-cmakecache ALL DEPENDS ${SIMDJSON_USER_CMAKECACHE})
|
||||
endif()
|
||||
|
||||
#
|
||||
# Set up test data
|
||||
#
|
||||
enable_testing()
|
||||
add_subdirectory(jsonchecker)
|
||||
add_subdirectory(jsonexamples)
|
||||
add_library(test-data INTERFACE)
|
||||
target_link_libraries(test-data INTERFACE jsonchecker-data jsonexamples-data)
|
||||
|
||||
#
|
||||
# Create the top level simdjson library (must be done at this level to use both src/ and include/
|
||||
# directories) and tools
|
||||
|
@ -146,16 +169,11 @@ add_subdirectory(include)
|
|||
add_subdirectory(src)
|
||||
add_subdirectory(windows)
|
||||
add_subdirectory(tools)
|
||||
add_subdirectory(singleheader)
|
||||
|
||||
#
|
||||
# Compile tools / tests / benchmarks
|
||||
#
|
||||
enable_testing()
|
||||
|
||||
add_library(test-data INTERFACE)
|
||||
target_compile_definitions(test-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/")
|
||||
target_compile_definitions(test-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/")
|
||||
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/twitter.json)
|
||||
|
||||
add_subdirectory(dependencies)
|
||||
add_subdirectory(tests)
|
||||
|
|
|
@ -31,5 +31,6 @@ Paul Dreik
|
|||
Jeremie Piotte
|
||||
Matthew Wilson
|
||||
Dušan Jovanović
|
||||
Matjaž Ostroveršnik
|
||||
# if you have contributed to the project and your name does not
|
||||
# appear in this list, please let us know!
|
||||
|
|
23
HACKING.md
23
HACKING.md
|
@ -20,7 +20,7 @@ simdjson's source structure, from the top level, looks like this:
|
|||
implementations).
|
||||
* simdjson.cpp: A "master source" that includes all implementation files from src/. This is
|
||||
equivalent to the distributed simdjson.cpp.
|
||||
* arm64/|fallback/|haswell/|westmere/: Architecture-specific implementations. All functions are
|
||||
* arm64/|fallback/|haswell/|westmere/: Architecture-specific implementations. All functions are
|
||||
Each architecture defines its own namespace, e.g. simdjson::haswell.
|
||||
* generic/: Generic implementations of the simdjson parser. These files may be included and
|
||||
compiled multiple times, from whichever architectures use them. They assume they are already
|
||||
|
@ -37,7 +37,7 @@ Other important files and directories:
|
|||
* **.drone.yml:** Definitions for Drone CI.
|
||||
* **.appveyor.yml:** Definitions for Appveyor CI (Windows).
|
||||
* **.circleci:** Definitions for Circle CI.
|
||||
* **amalgamation.sh:** Generates singleheader/simdjson.h and singleheader/simdjson.cpp for release.
|
||||
* **amalgamate.sh:** Generates singleheader/simdjson.h and singleheader/simdjson.cpp for release.
|
||||
* **benchmark:** This is where we do benchmarking. Benchmarking is core to every change we make; the
|
||||
cardinal rule is don't regress performance without knowing exactly why, and what you're trading
|
||||
for it. If you're not sure what else to do to check your performance, this is always a good start:
|
||||
|
@ -56,7 +56,7 @@ Other important files and directories:
|
|||
* **tools:** Source for executables that can be distributed with simdjson
|
||||
|
||||
> **Don't modify the files in singleheader/ directly; these are automatically generated.**
|
||||
>
|
||||
>
|
||||
> While we distribute those files on release, we *maintain* the files under include/ and src/.
|
||||
|
||||
While simdjson distributes just two files from the singleheader/ directory, we *maintain* the code in
|
||||
|
@ -73,7 +73,7 @@ you can regenerate them by running this at the top level:
|
|||
make amalgamate
|
||||
```
|
||||
|
||||
The amalgamator is at `amalgamation.sh` at the top level. It generates singleheader/simdjson.h by
|
||||
The amalgamator is at `amalgamate.sh` at the top level. It generates singleheader/simdjson.h by
|
||||
reading through include/simdjson.h, copy/pasting each header file into the amalgamated file at the
|
||||
point it gets included (but only once per header). singleheader/simdjson.cpp is generated from
|
||||
src/simdjson.cpp the same way, except files under generic/ may be included and copy/pasted multiple
|
||||
|
@ -157,6 +157,19 @@ make
|
|||
make test
|
||||
```
|
||||
|
||||
linux way:
|
||||
|
||||
```
|
||||
mkdir build # if necessary
|
||||
cd build
|
||||
cmake .. -DCMAKE_CXX_COMPILER=g++ # or
|
||||
cmake .. -DCMAKE_CXX_COMPILER=clang++
|
||||
make -j
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
### Usage (CMake on 64-bit Windows using Visual Studio)
|
||||
|
||||
We assume you have a common 64-bit Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later) or SSE 4.2 + CLMUL (2010 Westmere or later).
|
||||
|
@ -189,7 +202,7 @@ On Windows (64-bit):
|
|||
will build and install `simdjson` as a shared library.
|
||||
|
||||
```
|
||||
.\vcpkg.exe install simdjson:x64-windows-static
|
||||
.\vcpkg.exe install simdjson:x64-windows-static
|
||||
```
|
||||
|
||||
will build and install `simdjson` as a static library.
|
||||
|
|
13
Makefile
13
Makefile
|
@ -178,13 +178,16 @@ quicktests: run_basictests run_quickstart readme_examples readme_examples_noexce
|
|||
slowtests: run_testjson2json_sh run_issue150_sh
|
||||
|
||||
amalgamate:
|
||||
./amalgamation.sh
|
||||
singleheader/amalgamate.sh
|
||||
|
||||
singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamation_demo.cpp: amalgamation.sh src/simdjson.cpp $(SRCHEADERS) $(INCLUDEHEADERS)
|
||||
./amalgamation.sh
|
||||
singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp: singleheader/amalgamate.sh src/simdjson.cpp $(SRCHEADERS) $(INCLUDEHEADERS)
|
||||
singleheader/amalgamate.sh
|
||||
|
||||
singleheader/demo: singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamation_demo.cpp
|
||||
$(CXX) $(CXXFLAGS) -o singleheader/demo singleheader/amalgamation_demo.cpp -Isingleheader
|
||||
singleheader/demo: singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp
|
||||
$(CXX) $(CXXFLAGS) -o singleheader/demo singleheader/amalgamate_demo.cpp -Isingleheader
|
||||
|
||||
amalgamate_test: singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson
|
||||
singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson
|
||||
|
||||
submodules:
|
||||
-git submodule update --init --recursive
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
[![CircleCI](https://circleci.com/gh/simdjson/simdjson.svg?style=svg)](https://circleci.com/gh/simdjson/simdjson)
|
||||
[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/simdjson.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&q=proj%3Asimdjson&can=2)
|
||||
[![Build status](https://ci.appveyor.com/api/projects/status/ae77wp5v3lebmu6n/branch/master?svg=true)](https://ci.appveyor.com/project/lemire/simdjson-jmmti/branch/master)
|
||||
[![CirrusCI](https://api.cirrus-ci.com/github/simdjson/simdjson.svg)](https://cirrus-ci.com/github/simdjson/simdjson)
|
||||
[![][license img]][license]
|
||||
|
||||
simdjson : Parsing gigabytes of JSON per second
|
||||
|
|
|
@ -35,7 +35,7 @@ if (SIMDJSON_COMPETITION)
|
|||
target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER)
|
||||
endif()
|
||||
|
||||
if (NOT MSVC)
|
||||
IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
add_test(NAME checkperf
|
||||
COMMAND ${CMAKE_COMMAND} -E env
|
||||
CHECKPERF_REPOSITORY=https://github.com/simdjson/simdjson
|
||||
|
@ -46,4 +46,4 @@ if (NOT MSVC)
|
|||
set_property(TEST checkperf APPEND PROPERTY LABELS per_implementation)
|
||||
set_property(TEST checkperf APPEND PROPERTY DEPENDS parse perfdiff ${SIMDJSON_USER_CMAKECACHE})
|
||||
set_property(TEST checkperf PROPERTY RUN_SERIAL TRUE)
|
||||
endif()
|
||||
ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
|
|
|
@ -60,7 +60,7 @@ uint64_t global_rdtsc_overhead = (uint64_t)UINT64_MAX;
|
|||
do { \
|
||||
uint64_t cycles_start, cycles_final, cycles_diff; \
|
||||
uint64_t min_diff = UINT64_MAX; \
|
||||
for (int i = 0; i < repeat; i++) { \
|
||||
for (decltype(repeat) i = 0; i < repeat; i++) { \
|
||||
__asm volatile("" ::: /* pretend to clobber */ "memory"); \
|
||||
RDTSC_START(cycles_start); \
|
||||
test; \
|
||||
|
@ -73,7 +73,7 @@ uint64_t global_rdtsc_overhead = (uint64_t)UINT64_MAX;
|
|||
} while (0)
|
||||
|
||||
double diff(timespec start, timespec end) {
|
||||
return ((end.tv_nsec + 1000000000 * end.tv_sec) -
|
||||
return static_cast<double>((end.tv_nsec + 1000000000 * end.tv_sec) -
|
||||
(start.tv_nsec + 1000000000 * start.tv_sec)) /
|
||||
1000000000.0;
|
||||
}
|
||||
|
@ -100,7 +100,7 @@ double diff(timespec start, timespec end) {
|
|||
uint64_t sum_diff = 0; \
|
||||
double sumclockdiff = 0; \
|
||||
struct timespec time1, time2; \
|
||||
for (int i = 0; i < repeat; i++) { \
|
||||
for (decltype(repeat) i = 0; i < repeat; i++) { \
|
||||
pre; \
|
||||
__asm volatile("" ::: /* pretend to clobber */ "memory"); \
|
||||
clock_gettime(CLOCK_REALTIME, &time1); \
|
||||
|
@ -121,12 +121,12 @@ double diff(timespec start, timespec end) {
|
|||
sum_diff += cycles_diff; \
|
||||
} \
|
||||
uint64_t S = size; \
|
||||
float cycle_per_op = (min_diff) / (double)S; \
|
||||
float avg_cycle_per_op = (sum_diff) / ((double)S * repeat); \
|
||||
double cycle_per_op = static_cast<double>(min_diff) / static_cast<double>(S); \
|
||||
double avg_cycle_per_op = static_cast<double>(sum_diff) / (static_cast<double>(S) * static_cast<double>(repeat)); \
|
||||
double avg_gb_per_s = \
|
||||
((double)S * repeat) / ((sumclockdiff)*1000.0 * 1000.0 * 1000.0); \
|
||||
(static_cast<double>(S) * static_cast<double>(repeat)) / ((sumclockdiff)*1000.0 * 1000.0 * 1000.0); \
|
||||
double max_gb_per_s = \
|
||||
((double)S) / ((min_sumclockdiff)*1000.0 * 1000.0 * 1000.0); \
|
||||
static_cast<double>(S) / (min_sumclockdiff * 1000.0 * 1000.0 * 1000.0); \
|
||||
if (verbose) \
|
||||
printf(" %7.3f %s per input byte (best) ", cycle_per_op, unitname); \
|
||||
if (verbose) \
|
||||
|
@ -137,7 +137,7 @@ double diff(timespec start, timespec end) {
|
|||
if (verbose) \
|
||||
printf(" %13.0f documents/s (best)", 1.0/min_sumclockdiff); \
|
||||
if (verbose) \
|
||||
printf(" %13.0f documents/s (avg)", 1.0/(sumclockdiff/repeat)); \
|
||||
printf(" %13.0f documents/s (avg)", 1.0/(sumclockdiff/static_cast<double>(repeat))); \
|
||||
if (!verbose) \
|
||||
printf(" %20.3f %20.3f %20.3f %20.3f", cycle_per_op, \
|
||||
avg_cycle_per_op - cycle_per_op, max_gb_per_s, \
|
||||
|
@ -170,8 +170,8 @@ double diff(timespec start, timespec end) {
|
|||
sum_diff += cycles_diff; \
|
||||
} \
|
||||
uint64_t S = size; \
|
||||
float cycle_per_op = (min_diff) / (double)S; \
|
||||
float avg_cycle_per_op = (sum_diff) / ((double)S * repeat); \
|
||||
double cycle_per_op = static_cast<double>(min_diff) / static_cast<double>(S); \
|
||||
double avg_cycle_per_op = static_cast<double>(sum_diff) / (static_cast<double>(S) * static_cast<double>(repeat)); \
|
||||
if (verbose) \
|
||||
printf(" %.3f %s per input byte (best) ", cycle_per_op, unitname); \
|
||||
if (verbose) \
|
||||
|
|
|
@ -379,31 +379,31 @@ struct benchmarker {
|
|||
printf("%s%-13s: %8.4f ns per block (%6.2f%%) - %8.4f ns per byte - %8.4f ns per structural - %8.3f GB/s\n",
|
||||
prefix,
|
||||
"Speed",
|
||||
stage.elapsed_ns() / stats->blocks, // per block
|
||||
100.0 * stage.elapsed_sec() / all_stages.elapsed_sec(), // %
|
||||
stage.elapsed_ns() / stats->bytes, // per byte
|
||||
stage.elapsed_ns() / stats->structurals, // per structural
|
||||
(json.size() / 1000000000.0) / stage.elapsed_sec() // GB/s
|
||||
stage.elapsed_ns() / static_cast<double>(stats->blocks), // per block
|
||||
percent(stage.elapsed_sec(), all_stages.elapsed_sec()), // %
|
||||
stage.elapsed_ns() / static_cast<double>(stats->bytes), // per byte
|
||||
stage.elapsed_ns() / static_cast<double>(stats->structurals), // per structural
|
||||
(static_cast<double>(json.size()) / 1000000000.0) / stage.elapsed_sec() // GB/s
|
||||
);
|
||||
|
||||
if (collector.has_events()) {
|
||||
printf("%s%-13s: %8.4f per block (%6.2f%%) - %8.4f per byte - %8.4f per structural - %8.3f GHz est. frequency\n",
|
||||
prefix,
|
||||
"Cycles",
|
||||
stage.cycles() / stats->blocks,
|
||||
100.0 * stage.cycles() / all_stages.cycles(),
|
||||
stage.cycles() / stats->bytes,
|
||||
stage.cycles() / stats->structurals,
|
||||
stage.cycles() / static_cast<double>(stats->blocks),
|
||||
percent(stage.cycles(), all_stages.cycles()),
|
||||
stage.cycles() / static_cast<double>(stats->bytes),
|
||||
stage.cycles() / static_cast<double>(stats->structurals),
|
||||
(stage.cycles() / stage.elapsed_sec()) / 1000000000.0
|
||||
);
|
||||
printf("%s%-13s: %8.4f per block (%6.2f%%) - %8.4f per byte - %8.4f per structural - %8.3f per cycle\n",
|
||||
prefix,
|
||||
"Instructions",
|
||||
stage.instructions() / stats->blocks,
|
||||
100.0 * stage.instructions() / all_stages.instructions(),
|
||||
stage.instructions() / stats->bytes,
|
||||
stage.instructions() / stats->structurals,
|
||||
stage.instructions() / stage.cycles()
|
||||
stage.instructions() / static_cast<double>(stats->blocks),
|
||||
percent(stage.instructions(), all_stages.instructions()),
|
||||
stage.instructions() / static_cast<double>(stats->bytes),
|
||||
stage.instructions() / static_cast<double>(stats->structurals),
|
||||
stage.instructions() / static_cast<double>(stage.cycles())
|
||||
);
|
||||
|
||||
// NOTE: removed cycles/miss because it is a somewhat misleading stat
|
||||
|
@ -411,14 +411,21 @@ struct benchmarker {
|
|||
prefix,
|
||||
"Misses",
|
||||
stage.branch_misses(),
|
||||
100.0 * stage.branch_misses() / all_stages.branch_misses(),
|
||||
percent(stage.branch_misses(), all_stages.branch_misses()),
|
||||
stage.cache_misses(),
|
||||
100.0 * stage.cache_misses() / all_stages.cache_misses(),
|
||||
percent(stage.cache_misses(), all_stages.cache_misses()),
|
||||
stage.cache_references()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
static double percent(size_t a, size_t b) {
|
||||
return 100.0 * static_cast<double>(a) / static_cast<double>(b);
|
||||
}
|
||||
static double percent(double a, double b) {
|
||||
return 100.0 * a / b;
|
||||
}
|
||||
|
||||
void print(bool tabbed_output, size_t iterations) const {
|
||||
if (tabbed_output) {
|
||||
char* filename_copy = (char*)malloc(strlen(filename)+1);
|
||||
|
@ -432,14 +439,14 @@ struct benchmarker {
|
|||
base[strlen(base)-5] = '\0';
|
||||
}
|
||||
|
||||
double gb = json.size() / 1000000000.0;
|
||||
double gb = static_cast<double>(json.size()) / 1000000000.0;
|
||||
if (collector.has_events()) {
|
||||
printf("\"%s\"\t%f\t%f\t%f\t%f\t%f\t%f\t%f\n",
|
||||
base,
|
||||
allocate_stage.best.cycles() / json.size(),
|
||||
stage1.best.cycles() / json.size(),
|
||||
stage2.best.cycles() / json.size(),
|
||||
all_stages.best.cycles() / json.size(),
|
||||
allocate_stage.best.cycles() / static_cast<double>(json.size()),
|
||||
stage1.best.cycles() / static_cast<double>(json.size()),
|
||||
stage2.best.cycles() / static_cast<double>(json.size()),
|
||||
all_stages.best.cycles() / static_cast<double>(json.size()),
|
||||
gb / all_stages.best.elapsed_sec(),
|
||||
gb / stage1.best.elapsed_sec(),
|
||||
gb / stage2.best.elapsed_sec());
|
||||
|
@ -455,22 +462,22 @@ struct benchmarker {
|
|||
printf("\n");
|
||||
printf("%s\n", filename);
|
||||
printf("%s\n", string(strlen(filename), '=').c_str());
|
||||
printf("%9zu blocks - %10zu bytes - %5zu structurals (%5.1f %%)\n", stats->bytes / BYTES_PER_BLOCK, stats->bytes, stats->structurals, 100.0 * stats->structurals / stats->bytes);
|
||||
printf("%9zu blocks - %10zu bytes - %5zu structurals (%5.1f %%)\n", stats->bytes / BYTES_PER_BLOCK, stats->bytes, stats->structurals, percent(stats->structurals, stats->bytes));
|
||||
if (stats) {
|
||||
printf("special blocks with: utf8 %9zu (%5.1f %%) - escape %9zu (%5.1f %%) - 0 structurals %9zu (%5.1f %%) - 1+ structurals %9zu (%5.1f %%) - 8+ structurals %9zu (%5.1f %%) - 16+ structurals %9zu (%5.1f %%)\n",
|
||||
stats->blocks_with_utf8, 100.0 * stats->blocks_with_utf8 / stats->blocks,
|
||||
stats->blocks_with_escapes, 100.0 * stats->blocks_with_escapes / stats->blocks,
|
||||
stats->blocks_with_0_structurals, 100.0 * stats->blocks_with_0_structurals / stats->blocks,
|
||||
stats->blocks_with_1_structural, 100.0 * stats->blocks_with_1_structural / stats->blocks,
|
||||
stats->blocks_with_8_structurals, 100.0 * stats->blocks_with_8_structurals / stats->blocks,
|
||||
stats->blocks_with_16_structurals, 100.0 * stats->blocks_with_16_structurals / stats->blocks);
|
||||
stats->blocks_with_utf8, percent(stats->blocks_with_utf8, stats->blocks),
|
||||
stats->blocks_with_escapes, percent(stats->blocks_with_escapes, stats->blocks),
|
||||
stats->blocks_with_0_structurals, percent(stats->blocks_with_0_structurals, stats->blocks),
|
||||
stats->blocks_with_1_structural, percent(stats->blocks_with_1_structural, stats->blocks),
|
||||
stats->blocks_with_8_structurals, percent(stats->blocks_with_8_structurals, stats->blocks),
|
||||
stats->blocks_with_16_structurals, percent(stats->blocks_with_16_structurals, stats->blocks));
|
||||
printf("special block flips: utf8 %9zu (%5.1f %%) - escape %9zu (%5.1f %%) - 0 structurals %9zu (%5.1f %%) - 1+ structurals %9zu (%5.1f %%) - 8+ structurals %9zu (%5.1f %%) - 16+ structurals %9zu (%5.1f %%)\n",
|
||||
stats->blocks_with_utf8_flipped, 100.0 * stats->blocks_with_utf8_flipped / stats->blocks,
|
||||
stats->blocks_with_escapes_flipped, 100.0 * stats->blocks_with_escapes_flipped / stats->blocks,
|
||||
stats->blocks_with_0_structurals_flipped, 100.0 * stats->blocks_with_0_structurals_flipped / stats->blocks,
|
||||
stats->blocks_with_1_structural_flipped, 100.0 * stats->blocks_with_1_structural_flipped / stats->blocks,
|
||||
stats->blocks_with_8_structurals_flipped, 100.0 * stats->blocks_with_8_structurals_flipped / stats->blocks,
|
||||
stats->blocks_with_16_structurals_flipped, 100.0 * stats->blocks_with_16_structurals_flipped / stats->blocks);
|
||||
stats->blocks_with_utf8_flipped, percent(stats->blocks_with_utf8_flipped, stats->blocks),
|
||||
stats->blocks_with_escapes_flipped, percent(stats->blocks_with_escapes_flipped, stats->blocks),
|
||||
stats->blocks_with_0_structurals_flipped, percent(stats->blocks_with_0_structurals_flipped, stats->blocks),
|
||||
stats->blocks_with_1_structural_flipped, percent(stats->blocks_with_1_structural_flipped, stats->blocks),
|
||||
stats->blocks_with_8_structurals_flipped, percent(stats->blocks_with_8_structurals_flipped, stats->blocks),
|
||||
stats->blocks_with_16_structurals_flipped, percent(stats->blocks_with_16_structurals_flipped, stats->blocks));
|
||||
}
|
||||
printf("\n");
|
||||
printf("All Stages\n");
|
||||
|
@ -497,7 +504,7 @@ struct benchmarker {
|
|||
freqmin, freqmax, freqall);
|
||||
}
|
||||
}
|
||||
printf("\n%.1f documents parsed per second\n", iterations/loop.best.elapsed_sec());
|
||||
printf("\n%.1f documents parsed per second\n", static_cast<double>(iterations)/static_cast<double>(loop.best.elapsed_sec()));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
@ -366,7 +366,7 @@ int main(int argc, char *argv[]) {
|
|||
size_t size = s1.size();
|
||||
|
||||
int repeat = 500;
|
||||
int volume = p.size();
|
||||
size_t volume = p.size();
|
||||
if (just_data) {
|
||||
printf(
|
||||
"name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
|
||||
|
|
|
@ -56,11 +56,11 @@ struct event_count {
|
|||
|
||||
double elapsed_sec() const { return duration<double>(elapsed).count(); }
|
||||
double elapsed_ns() const { return duration<double, std::nano>(elapsed).count(); }
|
||||
double cycles() const { return event_counts[CPU_CYCLES]; }
|
||||
double instructions() const { return event_counts[INSTRUCTIONS]; }
|
||||
double branch_misses() const { return event_counts[BRANCH_MISSES]; }
|
||||
double cache_references() const { return event_counts[CACHE_REFERENCES]; }
|
||||
double cache_misses() const { return event_counts[CACHE_MISSES]; }
|
||||
double cycles() const { return static_cast<double>(event_counts[CPU_CYCLES]); }
|
||||
double instructions() const { return static_cast<double>(event_counts[INSTRUCTIONS]); }
|
||||
double branch_misses() const { return static_cast<double>(event_counts[BRANCH_MISSES]); }
|
||||
double cache_references() const { return static_cast<double>(event_counts[CACHE_REFERENCES]); }
|
||||
double cache_misses() const { return static_cast<double>(event_counts[CACHE_MISSES]); }
|
||||
|
||||
event_count& operator=(const event_count& other) {
|
||||
this->elapsed = other.elapsed;
|
||||
|
|
|
@ -12,7 +12,7 @@ double bench(std::string filename, simdjson::padded_string& p) {
|
|||
std::chrono::time_point<std::chrono::steady_clock> end_clock =
|
||||
std::chrono::steady_clock::now();
|
||||
std::chrono::duration<double> elapsed = end_clock - start_clock;
|
||||
return (p.size() / (1024. * 1024 * 1024.)) / elapsed.count();
|
||||
return (static_cast<double>(p.size()) / (1024. * 1024. * 1024.)) / elapsed.count();
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
|
@ -32,7 +32,7 @@ int main(int argc, char *argv[]) {
|
|||
double meanval = 0;
|
||||
double maxval = 0;
|
||||
double minval = 10000;
|
||||
std::cout << "file size: "<< (p.size() / (1024. * 1024 * 1024.)) << " GB" <<std::endl;
|
||||
std::cout << "file size: "<< (static_cast<double>(p.size()) / (1024. * 1024. * 1024.)) << " GB" <<std::endl;
|
||||
size_t times = p.size() > 1024*1024*1024 ? 5 : 50;
|
||||
#if __cpp_exceptions
|
||||
try {
|
||||
|
@ -49,7 +49,7 @@ std::cout << "file size: "<< (p.size() / (1024. * 1024 * 1024.)) << " GB" <<std
|
|||
return EXIT_FAILURE;
|
||||
}
|
||||
#endif
|
||||
std::cout << "average speed: " << meanval / times << " GB/s"<< std::endl;
|
||||
std::cout << "average speed: " << meanval / static_cast<double>(times) << " GB/s"<< std::endl;
|
||||
std::cout << "min speed : " << minval << " GB/s" << std::endl;
|
||||
std::cout << "max speed : " << maxval << " GB/s" << std::endl;
|
||||
return EXIT_SUCCESS;
|
||||
|
|
|
@ -18,7 +18,7 @@ template <int TYPE = PERF_TYPE_HARDWARE> class LinuxEvents {
|
|||
int fd;
|
||||
bool working;
|
||||
perf_event_attr attribs;
|
||||
int num_events;
|
||||
size_t num_events;
|
||||
std::vector<uint64_t> temp_result_vec;
|
||||
std::vector<uint64_t> ids;
|
||||
|
||||
|
@ -43,7 +43,7 @@ public:
|
|||
uint32_t i = 0;
|
||||
for (auto config : config_vec) {
|
||||
attribs.config = config;
|
||||
fd = syscall(__NR_perf_event_open, &attribs, pid, cpu, group, flags);
|
||||
fd = static_cast<int>(syscall(__NR_perf_event_open, &attribs, pid, cpu, group, flags));
|
||||
if (fd == -1) {
|
||||
report_error("perf_event_open");
|
||||
}
|
||||
|
|
|
@ -87,7 +87,7 @@ int main(int argc, char *argv[]) {
|
|||
buffer[p.size()] = '\0';
|
||||
|
||||
int repeat = 50;
|
||||
int volume = p.size();
|
||||
size_t volume = p.size();
|
||||
if (just_data) {
|
||||
printf(
|
||||
"name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
|
||||
|
@ -113,7 +113,7 @@ int main(int argc, char *argv[]) {
|
|||
|
||||
printf("minisize = %zu, original size = %zu (minified down to %.2f percent "
|
||||
"of original) \n",
|
||||
outlength, p.size(), outlength * 100.0 / p.size());
|
||||
outlength, p.size(), static_cast<double>(outlength) * 100.0 / static_cast<double>(p.size()));
|
||||
|
||||
/***
|
||||
* Is it worth it to minify before parsing?
|
||||
|
|
|
@ -56,7 +56,7 @@ int main (int argc, char *argv[]){
|
|||
auto end = std::chrono::steady_clock::now();
|
||||
|
||||
std::chrono::duration<double> secs = end - start;
|
||||
double speedinGBs = (p.size()) / (secs.count() * 1000000000.0);
|
||||
double speedinGBs = static_cast<double>(p.size()) / (static_cast<double>(secs.count()) * 1000000000.0);
|
||||
std::cout << speedinGBs << "\t\t\t\t" << count << std::endl;
|
||||
|
||||
if (parse_res != simdjson::SUCCESS) {
|
||||
|
@ -88,7 +88,7 @@ int main (int argc, char *argv[]){
|
|||
auto end = std::chrono::steady_clock::now();
|
||||
|
||||
std::chrono::duration<double> secs = end - start;
|
||||
double speedinGBs = (p.size()) / (secs.count() * 1000000000.0);
|
||||
double speedinGBs = static_cast<double>(p.size()) / (static_cast<double>(secs.count()) * 1000000000.0);
|
||||
if (speedinGBs > batch_size_res.at(i))
|
||||
batch_size_res[i] = speedinGBs;
|
||||
|
||||
|
@ -136,7 +136,7 @@ int main (int argc, char *argv[]){
|
|||
}
|
||||
|
||||
double min_result = *min_element(res.begin(), res.end());
|
||||
double speedinGBs = (p.size()) / (min_result * 1000000000.0);
|
||||
double speedinGBs = static_cast<double>(p.size()) / (min_result * 1000000000.0);
|
||||
|
||||
|
||||
std::cout << "Min: " << min_result << " bytes read: " << p.size()
|
||||
|
|
|
@ -336,7 +336,7 @@ int main(int argc, char *argv[]) {
|
|||
assert(stat_equal(s1, s2));
|
||||
assert(stat_equal(s1, s3));
|
||||
int repeat = 50;
|
||||
int volume = p.size();
|
||||
size_t volume = p.size();
|
||||
if (just_data) {
|
||||
printf("name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
|
||||
}
|
||||
|
|
|
@ -77,14 +77,14 @@ never_inline size_t sum_line_lengths(char * data, size_t length) {
|
|||
}
|
||||
|
||||
|
||||
bool bench(const char *filename, bool verbose, bool just_data, int repeat_multiplier) {
|
||||
bool bench(const char *filename, bool verbose, bool just_data, double repeat_multiplier) {
|
||||
auto [p, err] = simdjson::padded_string::load(filename);
|
||||
if (err) {
|
||||
std::cerr << "Could not load the file " << filename << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
int repeat = (50000000 * repeat_multiplier) / p.size();
|
||||
int repeat = static_cast<int>((50000000 * repeat_multiplier) / static_cast<double>(p.size()));
|
||||
if (repeat < 10) { repeat = 10; }
|
||||
|
||||
if (verbose) {
|
||||
|
@ -97,7 +97,7 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
|
|||
std::cout << p.size() << " B";
|
||||
std::cout << ": will run " << repeat << " iterations." << std::endl;
|
||||
}
|
||||
int volume = p.size();
|
||||
size_t volume = p.size();
|
||||
if (just_data) {
|
||||
printf("%-42s %20s %20s %20s %20s \n", "name", "cycles_per_byte",
|
||||
"cycles_per_byte_err", "gb_per_s", "gb_per_s_err");
|
||||
|
@ -214,7 +214,7 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
|
|||
buffer[p.size()] = '\0';
|
||||
BEST_TIME(
|
||||
"jsmn ",
|
||||
(jsmn_parse(&jparser, buffer, p.size(), tokens.get(), p.size()) > 0),
|
||||
(jsmn_parse(&jparser, buffer, p.size(), tokens.get(), static_cast<unsigned int>(p.size())) > 0),
|
||||
true, jsmn_init(&jparser), repeat, volume, !just_data);
|
||||
}
|
||||
memcpy(buffer, p.data(), p.size());
|
||||
|
@ -253,7 +253,7 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
|
|||
results.resize(evts.size());
|
||||
stats.resize(evts.size());
|
||||
std::fill(stats.begin(), stats.end(), 0); // unnecessary
|
||||
for (int i = 0; i < repeat; i++) {
|
||||
for (decltype(repeat) i = 0; i < repeat; i++) {
|
||||
unified.start();
|
||||
auto parse_error = parser.parse(p).error();
|
||||
if (parse_error)
|
||||
|
@ -265,13 +265,13 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
|
|||
printf("simdjson : cycles %10.0f instructions %10.0f branchmisses %10.0f "
|
||||
"cacheref %10.0f cachemisses %10.0f bytespercachemiss %10.0f "
|
||||
"inspercycle %10.1f insperbyte %10.1f\n",
|
||||
stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
|
||||
stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
|
||||
stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
|
||||
stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
|
||||
static_cast<double>(stats[0]) / static_cast<double>(repeat), static_cast<double>(stats[1]) / static_cast<double>(repeat),
|
||||
static_cast<double>(stats[2]) / static_cast<double>(repeat), static_cast<double>(stats[3]) / static_cast<double>(repeat),
|
||||
static_cast<double>(stats[4]) / static_cast<double>(repeat), static_cast<double>(volume) * static_cast<double>(repeat) / static_cast<double>(stats[2]),
|
||||
static_cast<double>(stats[1]) / static_cast<double>(stats[0]), static_cast<double>(stats[1]) / (static_cast<double>(volume) * static_cast<double>(repeat)));
|
||||
|
||||
std::fill(stats.begin(), stats.end(), 0);
|
||||
for (int i = 0; i < repeat; i++) {
|
||||
for (decltype(repeat) i = 0; i < repeat; i++) {
|
||||
memcpy(buffer, p.data(), p.size());
|
||||
buffer[p.size()] = '\0';
|
||||
unified.start();
|
||||
|
@ -285,13 +285,13 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
|
|||
printf("RapidJSON: cycles %10.0f instructions %10.0f branchmisses %10.0f "
|
||||
"cacheref %10.0f cachemisses %10.0f bytespercachemiss %10.0f "
|
||||
"inspercycle %10.1f insperbyte %10.1f\n",
|
||||
stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
|
||||
stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
|
||||
stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
|
||||
stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
|
||||
static_cast<double>(stats[0]) / static_cast<double>(repeat), static_cast<double>(stats[1]) / static_cast<double>(repeat),
|
||||
static_cast<double>(stats[2]) / static_cast<double>(repeat), static_cast<double>(stats[3]) / static_cast<double>(repeat),
|
||||
static_cast<double>(stats[4]) / static_cast<double>(repeat), static_cast<double>(volume) * static_cast<double>(repeat) / static_cast<double>(stats[2]),
|
||||
static_cast<double>(stats[1]) / static_cast<double>(stats[0]), static_cast<double>(stats[1]) / (static_cast<double>(volume) * static_cast<double>(repeat)));
|
||||
|
||||
std::fill(stats.begin(), stats.end(), 0); // unnecessary
|
||||
for (int i = 0; i < repeat; i++) {
|
||||
for (decltype(repeat) i = 0; i < repeat; i++) {
|
||||
memcpy(buffer, p.data(), p.size());
|
||||
unified.start();
|
||||
if (sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
|
||||
|
@ -305,10 +305,10 @@ bool bench(const char *filename, bool verbose, bool just_data, int repeat_multip
|
|||
printf("sajson : cycles %10.0f instructions %10.0f branchmisses %10.0f "
|
||||
"cacheref %10.0f cachemisses %10.0f bytespercachemiss %10.0f "
|
||||
"inspercycle %10.1f insperbyte %10.1f\n",
|
||||
stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
|
||||
stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
|
||||
stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
|
||||
stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
|
||||
static_cast<double>(stats[0]) / static_cast<double>(repeat), static_cast<double>(stats[1]) / static_cast<double>(repeat),
|
||||
static_cast<double>(stats[2]) / static_cast<double>(repeat), static_cast<double>(stats[3]) / static_cast<double>(repeat),
|
||||
static_cast<double>(stats[4]) / static_cast<double>(repeat), static_cast<double>(volume) * static_cast<double>(repeat) / static_cast<double>(stats[2]),
|
||||
static_cast<double>(stats[1]) / static_cast<double>(stats[0]), static_cast<double>(stats[1]) / (static_cast<double>(volume) * static_cast<double>(repeat)));
|
||||
|
||||
}
|
||||
#endif // __linux__
|
||||
|
|
|
@ -32,7 +32,7 @@ std::string exec(const char* cmd) {
|
|||
std::cerr << "popen() failed!" << std::endl;
|
||||
abort();
|
||||
}
|
||||
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
|
||||
while (fgets(buffer.data(), int(buffer.size()), pipe.get()) != nullptr) {
|
||||
result += buffer.data();
|
||||
}
|
||||
return result;
|
||||
|
@ -44,10 +44,10 @@ double readThroughput(std::string parseOutput) {
|
|||
double result = 0;
|
||||
int numResults = 0;
|
||||
while (std::getline(output, line)) {
|
||||
int pos = 0;
|
||||
std::string::size_type pos = 0;
|
||||
for (int i=0; i<5; i++) {
|
||||
pos = line.find('\t', pos);
|
||||
if (pos < 0) {
|
||||
if (pos == std::string::npos) {
|
||||
std::cerr << "Command printed out a line with less than 5 fields in it:\n" << line << std::endl;
|
||||
}
|
||||
pos++;
|
||||
|
|
|
@ -199,8 +199,8 @@ int main(int argc, char *argv[]) {
|
|||
std::cerr << "failure?" << std::endl;
|
||||
}
|
||||
}
|
||||
printf("%f %f %f %f ", cy1 * 1.0 / iterations, cl1 * 1.0 / iterations,
|
||||
cy2 * 1.0 / iterations, cl2 * 1.0 / iterations);
|
||||
printf("%f %f %f %f ", static_cast<double>(cy1) / static_cast<double>(iterations), static_cast<double>(cl1) / static_cast<double>(iterations),
|
||||
static_cast<double>(cy2) / static_cast<double>(iterations), static_cast<double>(cl2) / static_cast<double>(iterations));
|
||||
#endif // __linux__
|
||||
printf("\n");
|
||||
return EXIT_SUCCESS;
|
||||
|
|
|
@ -20,7 +20,7 @@ Requirements
|
|||
------------------
|
||||
|
||||
- A recent compiler (LLVM clang6 or better, GNU GCC 7 or better, Visual Studio 2017 or better). We require C++11 support as a minimum.
|
||||
- A 64-bit system (ARM or x64 Intel/AMD). Under Visual Studio, you must compile for 64-bit (e.g., x64) targets.
|
||||
- A 64-bit system (ARM or x64 Intel/AMD). We run tests on macOS, freeBSD, Linux and Windows. We recommend that Visual Studio users target a 64-bit build (x64).
|
||||
|
||||
Including simdjson
|
||||
------------------
|
||||
|
|
|
@ -3,7 +3,13 @@
|
|||
#
|
||||
|
||||
# TODO haven't quite decided the right way to run quickstart on Windows. Needs README update.
|
||||
if (NOT MSVC)
|
||||
#
|
||||
# Note: on macOS and other platforms, the 'command' described below may not work even if the cmake builds.
|
||||
# For example, it may be necessary to specify the sysroot, which CMake does, but the 'command' does not
|
||||
# handle such niceties. On a case-by-case basis it is fixable but it requires work that CMake knows how
|
||||
# to do but that is not trivial.
|
||||
#
|
||||
IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
# TODO run amalgamate first!
|
||||
function(add_quickstart_test TEST_NAME SOURCE_FILE)
|
||||
# Second argument is C++ standard name
|
||||
|
|
|
@ -105,6 +105,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
|||
// gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary
|
||||
#define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \
|
||||
|
@ -121,11 +122,49 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
|||
|
||||
#endif // MSC_VER
|
||||
|
||||
//
|
||||
// Backfill std::string_view using nonstd::string_view on C++11
|
||||
//
|
||||
#if (!SIMDJSON_CPLUSPLUS17)
|
||||
|
||||
|
||||
// C++17 requires string_view.
|
||||
#if SIMDJSON_CPLUSPLUS17
|
||||
#define SIMDJSON_HAS_STRING_VIEW
|
||||
#endif
|
||||
|
||||
// This macro (__cpp_lib_string_view) has to be defined
|
||||
// for C++17 and better, but if it is otherwise defined,
|
||||
// we are going to assume that string_view is available
|
||||
// even if we do not have C++17 support.
|
||||
#ifdef __cpp_lib_string_view
|
||||
#define SIMDJSON_HAS_STRING_VIEW
|
||||
#endif
|
||||
|
||||
// Some systems have string_view even if we do not have C++17 support,
|
||||
// and even if __cpp_lib_string_view is undefined, it is the case
|
||||
// with Apple clang version 11.
|
||||
// We must handle it. *This is important.*
|
||||
#ifndef SIMDJSON_HAS_STRING_VIEW
|
||||
#if defined __has_include
|
||||
// do not combine the next #if with the previous one (unsafe)
|
||||
#if __has_include (<string_view>)
|
||||
// now it is safe to trigger the include
|
||||
#include <string_view> // though the file is there, it does not follow that we got the implementation
|
||||
#if defined(_LIBCPP_STRING_VIEW)
|
||||
// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceeded C++17,
|
||||
// included string_view.
|
||||
// This means that we have string_view *even though* we may not have C++17.
|
||||
#define SIMDJSON_HAS_STRING_VIEW
|
||||
#endif // _LIBCPP_STRING_VIEW
|
||||
#endif // __has_include (<string_view>)
|
||||
#endif // defined __has_include
|
||||
#endif // def SIMDJSON_HAS_STRING_VIEW
|
||||
// end of complicated but important routine to try to detect string_view.
|
||||
|
||||
//
|
||||
// Backfill std::string_view using nonstd::string_view on systems where
|
||||
// we expect that string_view is missing. Important: if we get this wrong,
|
||||
// we will end up with two string_view definitions and potential trouble.
|
||||
// That is why we work so hard above to avoid it.
|
||||
//
|
||||
#ifndef SIMDJSON_HAS_STRING_VIEW
|
||||
SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
|
||||
#include "simdjson/nonstd/string_view.hpp"
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
@ -133,6 +172,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
|||
namespace std {
|
||||
using string_view = nonstd::string_view;
|
||||
}
|
||||
#endif // if (SIMDJSON_CPLUSPLUS < 201703L)
|
||||
#endif // SIMDJSON_HAS_STRING_VIEW
|
||||
#undef SIMDJSON_HAS_STRING_VIEW // We are not going to need this macro anymore.
|
||||
|
||||
#endif // SIMDJSON_COMMON_DEFS_H
|
||||
|
|
|
@ -124,8 +124,9 @@ public:
|
|||
* Get the next value.
|
||||
*
|
||||
* Part of the std::iterator interface.
|
||||
*
|
||||
*/
|
||||
inline void operator++() noexcept;
|
||||
inline iterator& operator++() noexcept;
|
||||
/**
|
||||
* Check if these values come from the same place in the JSON.
|
||||
*
|
||||
|
@ -205,8 +206,9 @@ public:
|
|||
* Get the next key/value pair.
|
||||
*
|
||||
* Part of the std::iterator interface.
|
||||
*
|
||||
*/
|
||||
inline void operator++() noexcept;
|
||||
inline iterator& operator++() noexcept;
|
||||
/**
|
||||
* Check if these key value pairs come from the same place in the JSON.
|
||||
*
|
||||
|
@ -373,13 +375,13 @@ public:
|
|||
bool dump_raw_tape(std::ostream &os) const noexcept;
|
||||
|
||||
/** @private Structural values. */
|
||||
std::unique_ptr<uint64_t[]> tape;
|
||||
std::unique_ptr<uint64_t[]> tape{};
|
||||
|
||||
/** @private String values.
|
||||
*
|
||||
* Should be at least byte_capacity.
|
||||
*/
|
||||
std::unique_ptr<uint8_t[]> string_buf;
|
||||
std::unique_ptr<uint8_t[]> string_buf{};
|
||||
|
||||
private:
|
||||
inline error_code allocate(size_t len) noexcept;
|
||||
|
@ -660,8 +662,7 @@ public:
|
|||
* to allocate an initial capacity, call allocate() after constructing the parser.
|
||||
* Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process).
|
||||
*/
|
||||
really_inline parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept;
|
||||
|
||||
really_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept;
|
||||
/**
|
||||
* Take another parser's buffers and state.
|
||||
*
|
||||
|
@ -811,7 +812,7 @@ public:
|
|||
* - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
|
||||
* - other json errors if parsing fails.
|
||||
*/
|
||||
inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
|
||||
inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
|
||||
|
||||
/**
|
||||
* Parse a buffer containing many JSON documents.
|
||||
|
@ -953,21 +954,21 @@ public:
|
|||
/** @private Number of structural indices passed from stage 1 to stage 2 */
|
||||
uint32_t n_structural_indexes{0};
|
||||
/** @private Structural indices passed from stage 1 to stage 2 */
|
||||
std::unique_ptr<uint32_t[]> structural_indexes;
|
||||
std::unique_ptr<uint32_t[]> structural_indexes{};
|
||||
|
||||
/** @private Tape location of each open { or [ */
|
||||
std::unique_ptr<scope_descriptor[]> containing_scope;
|
||||
std::unique_ptr<scope_descriptor[]> containing_scope{};
|
||||
|
||||
#ifdef SIMDJSON_USE_COMPUTED_GOTO
|
||||
/** @private Return address of each open { or [ */
|
||||
std::unique_ptr<void*[]> ret_address;
|
||||
std::unique_ptr<void*[]> ret_address{};
|
||||
#else
|
||||
/** @private Return address of each open { or [ */
|
||||
std::unique_ptr<char[]> ret_address;
|
||||
std::unique_ptr<char[]> ret_address{};
|
||||
#endif
|
||||
|
||||
/** @private Next write location in the string buf for stage 2 parsing */
|
||||
uint8_t *current_string_buf_loc;
|
||||
uint8_t *current_string_buf_loc{};
|
||||
|
||||
/** @private Use `if (parser.parse(...).error())` instead */
|
||||
bool valid{false};
|
||||
|
@ -975,7 +976,7 @@ public:
|
|||
error_code error{UNINITIALIZED};
|
||||
|
||||
/** @private Use `parser.parse(...).value()` instead */
|
||||
document doc;
|
||||
document doc{};
|
||||
|
||||
/** @private returns true if the document parsed was valid */
|
||||
[[deprecated("Use the result of parser.parse() instead")]]
|
||||
|
|
|
@ -135,8 +135,8 @@ private:
|
|||
error_code error{SUCCESS_AND_HAS_MORE};
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
error_code stage1_is_ok_thread{SUCCESS};
|
||||
std::thread stage_1_thread;
|
||||
dom::parser parser_thread;
|
||||
std::thread stage_1_thread{};
|
||||
dom::parser parser_thread{};
|
||||
#endif
|
||||
friend class dom::parser;
|
||||
}; // class document_stream
|
||||
|
|
|
@ -17,6 +17,7 @@ namespace simdjson {
|
|||
*/
|
||||
class implementation {
|
||||
public:
|
||||
|
||||
/**
|
||||
* The name of this implementation.
|
||||
*
|
||||
|
@ -131,6 +132,7 @@ protected:
|
|||
_required_instruction_sets(required_instruction_sets)
|
||||
{
|
||||
}
|
||||
virtual ~implementation()=default;
|
||||
|
||||
private:
|
||||
/**
|
||||
|
|
|
@ -213,7 +213,7 @@ inline error_code document::allocate(size_t capacity) noexcept {
|
|||
|
||||
// a pathological input like "[[[[..." would generate len tape elements, so
|
||||
// need a capacity of at least len + 1, but it is also possible to do
|
||||
// worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"
|
||||
// worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"
|
||||
//where len + 1 tape elements are
|
||||
// generated, see issue https://github.com/lemire/simdjson/issues/345
|
||||
size_t tape_capacity = ROUNDUP_N(capacity + 2, 64);
|
||||
|
@ -229,12 +229,12 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
|
|||
uint32_t string_length;
|
||||
size_t tape_idx = 0;
|
||||
uint64_t tape_val = tape[tape_idx];
|
||||
uint8_t type = (tape_val >> 56);
|
||||
uint8_t type = uint8_t(tape_val >> 56);
|
||||
os << tape_idx << " : " << type;
|
||||
tape_idx++;
|
||||
size_t how_many = 0;
|
||||
if (type == 'r') {
|
||||
how_many = tape_val & internal::JSON_VALUE_MASK;
|
||||
how_many = size_t(tape_val & internal::JSON_VALUE_MASK);
|
||||
} else {
|
||||
// Error: no starting root node?
|
||||
return false;
|
||||
|
@ -245,7 +245,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
|
|||
os << tape_idx << " : ";
|
||||
tape_val = tape[tape_idx];
|
||||
payload = tape_val & internal::JSON_VALUE_MASK;
|
||||
type = (tape_val >> 56);
|
||||
type = uint8_t(tape_val >> 56);
|
||||
switch (type) {
|
||||
case '"': // we have a string
|
||||
os << "string \"";
|
||||
|
@ -312,7 +312,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
|
|||
}
|
||||
tape_val = tape[tape_idx];
|
||||
payload = tape_val & internal::JSON_VALUE_MASK;
|
||||
type = (tape_val >> 56);
|
||||
type = uint8_t(tape_val >> 56);
|
||||
os << tape_idx << " : " << type << "\t// pointing to " << payload
|
||||
<< " (start root)\n";
|
||||
return true;
|
||||
|
@ -322,7 +322,9 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
|
|||
// parser inline implementation
|
||||
//
|
||||
really_inline parser::parser(size_t max_capacity) noexcept
|
||||
: _max_capacity{max_capacity}, loaded_bytes(nullptr, &aligned_free_char) {}
|
||||
: _max_capacity{max_capacity},
|
||||
loaded_bytes(nullptr, &aligned_free_char)
|
||||
{}
|
||||
inline bool parser::is_valid() const noexcept { return valid; }
|
||||
inline int parser::get_error_code() const noexcept { return error; }
|
||||
inline std::string parser::get_error_message() const noexcept { return error_message(error); }
|
||||
|
@ -471,7 +473,7 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept {
|
|||
//
|
||||
// Initialize stage 1 output
|
||||
//
|
||||
uint32_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7;
|
||||
size_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7;
|
||||
structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); // TODO realloc
|
||||
if (!structural_indexes) {
|
||||
return MEMALLOC;
|
||||
|
@ -565,7 +567,7 @@ inline simdjson_result<element> array::at(const std::string_view &json_pointer)
|
|||
size_t array_index = 0;
|
||||
size_t i;
|
||||
for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) {
|
||||
uint8_t digit = uint8_t(json_pointer[i]) - '0';
|
||||
uint8_t digit = uint8_t(json_pointer[i] - '0');
|
||||
// Check for non-digit in array index. If it's there, we're trying to get a field in an object
|
||||
if (digit > 9) { return INCORRECT_TYPE; }
|
||||
array_index = array_index*10 + digit;
|
||||
|
@ -604,8 +606,9 @@ inline element array::iterator::operator*() const noexcept {
|
|||
inline bool array::iterator::operator!=(const array::iterator& other) const noexcept {
|
||||
return json_index != other.json_index;
|
||||
}
|
||||
inline void array::iterator::operator++() noexcept {
|
||||
inline array::iterator& array::iterator::operator++() noexcept {
|
||||
json_index = after_element();
|
||||
return *this;
|
||||
}
|
||||
|
||||
//
|
||||
|
@ -703,12 +706,13 @@ inline const key_value_pair object::iterator::operator*() const noexcept {
|
|||
inline bool object::iterator::operator!=(const object::iterator& other) const noexcept {
|
||||
return json_index != other.json_index;
|
||||
}
|
||||
inline void object::iterator::operator++() noexcept {
|
||||
inline object::iterator& object::iterator::operator++() noexcept {
|
||||
json_index++;
|
||||
json_index = after_element();
|
||||
return *this;
|
||||
}
|
||||
inline std::string_view object::iterator::key() const noexcept {
|
||||
size_t string_buf_index = tape_value();
|
||||
size_t string_buf_index = size_t(tape_value());
|
||||
uint32_t len;
|
||||
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
|
||||
return std::string_view(
|
||||
|
@ -717,7 +721,7 @@ inline std::string_view object::iterator::key() const noexcept {
|
|||
);
|
||||
}
|
||||
inline const char* object::iterator::key_c_str() const noexcept {
|
||||
return reinterpret_cast<const char *>(&doc->string_buf[tape_value() + sizeof(uint32_t)]);
|
||||
return reinterpret_cast<const char *>(&doc->string_buf[size_t(tape_value()) + sizeof(uint32_t)]);
|
||||
}
|
||||
inline element object::iterator::value() const noexcept {
|
||||
return element(doc, json_index + 1);
|
||||
|
@ -757,7 +761,7 @@ template<>
|
|||
inline simdjson_result<const char *> element::get<const char *>() const noexcept {
|
||||
switch (tape_ref_type()) {
|
||||
case internal::tape_type::STRING: {
|
||||
size_t string_buf_index = tape_value();
|
||||
size_t string_buf_index = size_t(tape_value());
|
||||
return reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]);
|
||||
}
|
||||
default:
|
||||
|
@ -781,7 +785,7 @@ inline simdjson_result<uint64_t> element::get<uint64_t>() const noexcept {
|
|||
if (result < 0) {
|
||||
return NUMBER_OUT_OF_RANGE;
|
||||
}
|
||||
return static_cast<uint64_t>(result);
|
||||
return uint64_t(result);
|
||||
}
|
||||
return INCORRECT_TYPE;
|
||||
}
|
||||
|
@ -815,9 +819,9 @@ inline simdjson_result<double> element::get<double>() const noexcept {
|
|||
// information. (This could also be solved with profile-guided optimization.)
|
||||
if(unlikely(!is_double())) { // branch rarely taken
|
||||
if(is_uint64()) {
|
||||
return next_tape_value<uint64_t>();
|
||||
return double(next_tape_value<uint64_t>());
|
||||
} else if(is_int64()) {
|
||||
return next_tape_value<int64_t>();
|
||||
return double(next_tape_value<int64_t>());
|
||||
}
|
||||
return INCORRECT_TYPE;
|
||||
}
|
||||
|
@ -1096,27 +1100,27 @@ really_inline tape_ref::tape_ref(const document *_doc, size_t _json_index) noexc
|
|||
// most significant 8 bits.
|
||||
|
||||
really_inline bool tape_ref::is_double() const noexcept {
|
||||
constexpr uint64_t tape_double = static_cast<uint64_t>(tape_type::DOUBLE)<<56;
|
||||
constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56;
|
||||
return doc->tape[json_index] == tape_double;
|
||||
}
|
||||
really_inline bool tape_ref::is_int64() const noexcept {
|
||||
constexpr uint64_t tape_int64 = static_cast<uint64_t>(tape_type::INT64)<<56;
|
||||
constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56;
|
||||
return doc->tape[json_index] == tape_int64;
|
||||
}
|
||||
really_inline bool tape_ref::is_uint64() const noexcept {
|
||||
constexpr uint64_t tape_uint64 = static_cast<uint64_t>(tape_type::UINT64)<<56;
|
||||
constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56;
|
||||
return doc->tape[json_index] == tape_uint64;
|
||||
}
|
||||
really_inline bool tape_ref::is_false() const noexcept {
|
||||
constexpr uint64_t tape_false = static_cast<uint64_t>(tape_type::FALSE_VALUE)<<56;
|
||||
constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56;
|
||||
return doc->tape[json_index] == tape_false;
|
||||
}
|
||||
really_inline bool tape_ref::is_true() const noexcept {
|
||||
constexpr uint64_t tape_true = static_cast<uint64_t>(tape_type::TRUE_VALUE)<<56;
|
||||
constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56;
|
||||
return doc->tape[json_index] == tape_true;
|
||||
}
|
||||
really_inline bool tape_ref::is_null_on_tape() const noexcept {
|
||||
constexpr uint64_t tape_null = static_cast<uint64_t>(tape_type::NULL_VALUE)<<56;
|
||||
constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56;
|
||||
return doc->tape[json_index] == tape_null;
|
||||
}
|
||||
|
||||
|
@ -1140,10 +1144,10 @@ really_inline uint64_t internal::tape_ref::tape_value() const noexcept {
|
|||
return doc->tape[json_index] & internal::JSON_VALUE_MASK;
|
||||
}
|
||||
really_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept {
|
||||
return static_cast<uint32_t>(doc->tape[json_index]);
|
||||
return uint32_t(doc->tape[json_index]);
|
||||
}
|
||||
really_inline uint32_t internal::tape_ref::scope_count() const noexcept {
|
||||
return static_cast<uint32_t>((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK);
|
||||
return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
|
@ -1158,7 +1162,7 @@ really_inline T tape_ref::next_tape_value() const noexcept {
|
|||
return x;
|
||||
}
|
||||
inline std::string_view internal::tape_ref::get_string_view() const noexcept {
|
||||
size_t string_buf_index = tape_value();
|
||||
size_t string_buf_index = size_t(tape_value());
|
||||
uint32_t len;
|
||||
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
|
||||
return std::string_view(
|
||||
|
|
|
@ -33,7 +33,7 @@ namespace internal {
|
|||
* complete
|
||||
* document, therefore the last json buffer location is the end of the batch
|
||||
* */
|
||||
inline size_t find_last_json_buf_idx(const uint8_t *buf, size_t size, const dom::parser &parser) {
|
||||
inline uint32_t find_last_json_buf_idx(const uint8_t *buf, size_t size, const dom::parser &parser) {
|
||||
// this function can be generally useful
|
||||
if (parser.n_structural_indexes == 0)
|
||||
return 0;
|
||||
|
@ -85,7 +85,7 @@ static inline bool is_ascii(char c) {
|
|||
return ((unsigned char)c) <= 127;
|
||||
}
|
||||
|
||||
// if the string ends with UTF-8 values, backtrack
|
||||
// if the string ends with UTF-8 values, backtrack
|
||||
// up to the first ASCII character. May return 0.
|
||||
static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) {
|
||||
while ((len > 0) and (not is_ascii(c[len - 1]))) {
|
||||
|
@ -100,14 +100,19 @@ static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) {
|
|||
|
||||
namespace simdjson {
|
||||
namespace dom {
|
||||
|
||||
really_inline document_stream::document_stream(
|
||||
dom::parser &_parser,
|
||||
const uint8_t *buf,
|
||||
size_t len,
|
||||
size_t batch_size,
|
||||
error_code _error
|
||||
) noexcept : parser{_parser}, _buf{buf}, _len{len}, _batch_size(batch_size), error{_error} {
|
||||
) noexcept
|
||||
: parser{_parser},
|
||||
_buf{buf},
|
||||
_len{len},
|
||||
_batch_size(batch_size),
|
||||
error(_error)
|
||||
{
|
||||
if (!error) { error = json_parse(); }
|
||||
}
|
||||
|
||||
|
@ -172,7 +177,7 @@ inline error_code document_stream::json_parse() noexcept {
|
|||
if (stage1_is_ok != simdjson::SUCCESS) {
|
||||
return stage1_is_ok;
|
||||
}
|
||||
size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
|
||||
uint32_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
|
||||
if (last_index == 0) {
|
||||
if (parser.n_structural_indexes == 0) {
|
||||
return simdjson::EMPTY;
|
||||
|
@ -249,7 +254,7 @@ inline error_code document_stream::json_parse() noexcept {
|
|||
if (stage1_is_ok != simdjson::SUCCESS) {
|
||||
return stage1_is_ok;
|
||||
}
|
||||
size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
|
||||
uint32_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
|
||||
if (last_index == 0) {
|
||||
if (parser.n_structural_indexes == 0) {
|
||||
return EMPTY;
|
||||
|
|
|
@ -5,6 +5,10 @@
|
|||
|
||||
namespace simdjson {
|
||||
|
||||
// VS2017 reports deprecated warnings when you define a deprecated class's methods.
|
||||
SIMDJSON_PUSH_DISABLE_WARNINGS
|
||||
SIMDJSON_DISABLE_DEPRECATED_WARNING
|
||||
|
||||
// Because of template weirdness, the actual class definition is inline in the document class
|
||||
|
||||
WARN_UNUSED bool dom::parser::Iterator::is_ok() const {
|
||||
|
@ -54,7 +58,7 @@ bool dom::parser::Iterator::move_forward() {
|
|||
|
||||
location += 1;
|
||||
current_val = doc.tape[location];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -62,7 +66,7 @@ void dom::parser::Iterator::move_to_value() {
|
|||
// assume that we are on a key, so move by 1.
|
||||
location += 1;
|
||||
current_val = doc.tape[location];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
}
|
||||
|
||||
bool dom::parser::Iterator::move_to_key(const char *key) {
|
||||
|
@ -139,14 +143,14 @@ bool dom::parser::Iterator::prev() {
|
|||
oldnpos = npos;
|
||||
if ((current_type == '[') || (current_type == '{')) {
|
||||
// we need to jump
|
||||
npos = static_cast<uint32_t>(current_val);
|
||||
npos = uint32_t(current_val);
|
||||
} else {
|
||||
npos = npos + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
|
||||
}
|
||||
} while (npos < target_location);
|
||||
location = oldnpos;
|
||||
current_val = doc.tape[location];
|
||||
current_type = current_val >> 56;
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -159,7 +163,7 @@ bool dom::parser::Iterator::up() {
|
|||
depth--;
|
||||
location -= 1;
|
||||
current_val = doc.tape[location];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -168,7 +172,7 @@ bool dom::parser::Iterator::down() {
|
|||
return false;
|
||||
}
|
||||
if ((current_type == '[') || (current_type == '{')) {
|
||||
size_t npos = static_cast<uint32_t>(current_val);
|
||||
size_t npos = uint32_t(current_val);
|
||||
if (npos == location + 2) {
|
||||
return false; // we have an empty scope
|
||||
}
|
||||
|
@ -178,7 +182,7 @@ bool dom::parser::Iterator::down() {
|
|||
depth_index[depth].start_of_scope = location;
|
||||
depth_index[depth].scope_type = current_type;
|
||||
current_val = doc.tape[location];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
@ -187,19 +191,19 @@ bool dom::parser::Iterator::down() {
|
|||
void dom::parser::Iterator::to_start_scope() {
|
||||
location = depth_index[depth].start_of_scope;
|
||||
current_val = doc.tape[location];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
}
|
||||
|
||||
bool dom::parser::Iterator::next() {
|
||||
size_t npos;
|
||||
if ((current_type == '[') || (current_type == '{')) {
|
||||
// we need to jump
|
||||
npos = static_cast<uint32_t>(current_val);
|
||||
npos = uint32_t(current_val);
|
||||
} else {
|
||||
npos = location + (is_number() ? 2 : 1);
|
||||
}
|
||||
uint64_t next_val = doc.tape[npos];
|
||||
uint8_t next_type = (next_val >> 56);
|
||||
uint8_t next_type = uint8_t(next_val >> 56);
|
||||
if ((next_type == ']') || (next_type == '}')) {
|
||||
return false; // we reached the end of the scope
|
||||
}
|
||||
|
@ -208,9 +212,9 @@ bool dom::parser::Iterator::next() {
|
|||
current_type = next_type;
|
||||
return true;
|
||||
}
|
||||
|
||||
dom::parser::Iterator::Iterator(const dom::parser &pj) noexcept(false)
|
||||
: doc(pj.doc), depth(0), location(0), tape_length(0) {
|
||||
: doc(pj.doc)
|
||||
{
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
if (!pj.valid) { throw simdjson_error(pj.error); }
|
||||
#else
|
||||
|
@ -221,26 +225,31 @@ dom::parser::Iterator::Iterator(const dom::parser &pj) noexcept(false)
|
|||
depth_index = new scopeindex_t[max_depth + 1];
|
||||
depth_index[0].start_of_scope = location;
|
||||
current_val = doc.tape[location++];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
depth_index[0].scope_type = current_type;
|
||||
tape_length = current_val & internal::JSON_VALUE_MASK;
|
||||
tape_length = size_t(current_val & internal::JSON_VALUE_MASK);
|
||||
if (location < tape_length) {
|
||||
// If we make it here, then depth_capacity must >=2, but the compiler
|
||||
// may not know this.
|
||||
current_val = doc.tape[location];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
depth++;
|
||||
assert(depth < max_depth);
|
||||
depth_index[depth].start_of_scope = location;
|
||||
depth_index[depth].scope_type = current_type;
|
||||
}
|
||||
}
|
||||
|
||||
dom::parser::Iterator::Iterator(
|
||||
const dom::parser::Iterator &o) noexcept
|
||||
: doc(o.doc), max_depth(o.depth), depth(o.depth), location(o.location),
|
||||
tape_length(o.tape_length), current_type(o.current_type),
|
||||
current_val(o.current_val) {
|
||||
: doc(o.doc),
|
||||
max_depth(o.depth),
|
||||
depth(o.depth),
|
||||
location(o.location),
|
||||
tape_length(o.tape_length),
|
||||
current_type(o.current_type),
|
||||
current_val(o.current_val),
|
||||
depth_index()
|
||||
{
|
||||
depth_index = new scopeindex_t[max_depth+1];
|
||||
memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0]));
|
||||
}
|
||||
|
@ -287,7 +296,7 @@ bool dom::parser::Iterator::print(std::ostream &os, bool escape_strings) const {
|
|||
case '}': // we end an object
|
||||
case '[': // we start an array
|
||||
case ']': // we end an array
|
||||
os << static_cast<char>(current_type);
|
||||
os << char(current_type);
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
|
@ -314,7 +323,7 @@ bool dom::parser::Iterator::move_to(const char *pointer,
|
|||
new_pointer[new_length] = '\\';
|
||||
new_length++;
|
||||
}
|
||||
new_pointer[new_length] = fragment;
|
||||
new_pointer[new_length] = char(fragment);
|
||||
i += 3;
|
||||
#if __cpp_exceptions
|
||||
} catch (std::invalid_argument &) {
|
||||
|
@ -427,7 +436,7 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
|
|||
|
||||
bool found = false;
|
||||
if (is_object()) {
|
||||
if (move_to_key(key_or_index.c_str(), key_or_index.length())) {
|
||||
if (move_to_key(key_or_index.c_str(), uint32_t(key_or_index.length()))) {
|
||||
found = relative_move_to(pointer + offset, length - offset);
|
||||
}
|
||||
} else if (is_array()) {
|
||||
|
@ -439,14 +448,14 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
|
|||
size_t npos;
|
||||
if ((current_type == '[') || (current_type == '{')) {
|
||||
// we need to jump
|
||||
npos = static_cast<uint32_t>(current_val);
|
||||
npos = uint32_t(current_val);
|
||||
} else {
|
||||
npos =
|
||||
location + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
|
||||
}
|
||||
location = npos;
|
||||
current_val = doc.tape[npos];
|
||||
current_type = (current_val >> 56);
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
return true; // how could it fail ?
|
||||
}
|
||||
} else { // regular numeric index
|
||||
|
@ -469,6 +478,8 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
|
|||
return found;
|
||||
}
|
||||
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
|
||||
|
|
|
@ -49,7 +49,7 @@ inline std::ostream& operator<<(std::ostream& out, const escape_json_string &une
|
|||
if ((unsigned char)unescaped.str[i] <= 0x1F) {
|
||||
// TODO can this be done once at the beginning, or will it mess up << char?
|
||||
std::ios::fmtflags f(out.flags());
|
||||
out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(unescaped.str[i]);
|
||||
out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << int(unescaped.str[i]);
|
||||
out.flags(f);
|
||||
} else {
|
||||
out << unescaped.str[i];
|
||||
|
|
|
@ -22,6 +22,8 @@ public:
|
|||
inline Iterator(const Iterator &o) noexcept;
|
||||
inline ~Iterator() noexcept;
|
||||
|
||||
inline Iterator& operator=(const Iterator&) = delete;
|
||||
|
||||
inline bool is_ok() const;
|
||||
|
||||
// useful for debugging purposes
|
||||
|
@ -187,7 +189,7 @@ public:
|
|||
// is referenced is undefined, and evaluation fails". Here we just return
|
||||
// the first corresponding value.
|
||||
inline bool move_to(const std::string &pointer) {
|
||||
return move_to(pointer.c_str(), pointer.length());
|
||||
return move_to(pointer.c_str(), uint32_t(pointer.length()));
|
||||
}
|
||||
|
||||
private:
|
||||
|
@ -253,13 +255,13 @@ public:
|
|||
|
||||
private:
|
||||
const document &doc;
|
||||
size_t max_depth;
|
||||
size_t depth;
|
||||
size_t location; // our current location on a tape
|
||||
size_t tape_length;
|
||||
uint8_t current_type;
|
||||
uint64_t current_val;
|
||||
scopeindex_t *depth_index;
|
||||
size_t max_depth{};
|
||||
size_t depth{};
|
||||
size_t location{}; // our current location on a tape
|
||||
size_t tape_length{};
|
||||
uint8_t current_type{};
|
||||
uint64_t current_val{};
|
||||
scopeindex_t *depth_index{};
|
||||
};
|
||||
|
||||
} // namespace simdjson
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
set(SIMDJSON_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
|
||||
add_library(jsonchecker-data INTERFACE)
|
||||
target_compile_definitions(jsonchecker-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
|
|
@ -0,0 +1,5 @@
|
|||
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
|
||||
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/twitter.json PARENT_SCOPE)
|
||||
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_SOURCE_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
|
||||
add_library(jsonexamples-data INTERFACE)
|
||||
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
|
|
@ -0,0 +1,26 @@
|
|||
#
|
||||
# Amalgamation
|
||||
#
|
||||
# We should check whether bash is available here and avoid failures on systems where
|
||||
# bash is unavailable.
|
||||
if (NOT MSVC)
|
||||
##
|
||||
# Important! The script amalgamate.sh is not generally executable. It
|
||||
# assumes that bash is at /bin/bash which may not be true.
|
||||
###
|
||||
set(SINGLEHEADER_FILES simdjson.h simdjson.cpp amalgamate_demo.cpp README.md)
|
||||
add_custom_command(
|
||||
OUTPUT ${SINGLEHEADER_FILES}
|
||||
COMMAND ${CMAKE_COMMAND} -E env
|
||||
AMALGAMATE_SOURCE_PATH=${PROJECT_SOURCE_DIR}/src
|
||||
AMALGAMATE_INPUT_PATH=${PROJECT_SOURCE_DIR}/include
|
||||
AMALGAMATE_OUTPUT_PATH=${CMAKE_CURRENT_BINARY_DIR}
|
||||
bash ${CMAKE_CURRENT_SOURCE_DIR}/amalgamate.sh
|
||||
DEPENDS amalgamate.sh simdjson-source
|
||||
)
|
||||
|
||||
add_custom_target(amalgamate DEPENDS ${SINGLEHEADER_FILES})
|
||||
add_executable(amalgamate_demo amalgamate_demo.cpp)
|
||||
target_link_libraries(amalgamate_demo simdjson-include-source)
|
||||
add_test(amalgamate_demo amalgamate_demo ${EXAMPLE_JSON} ${EXAMPLE_NDJSON})
|
||||
endif()
|
|
@ -1 +1,2 @@
|
|||
c++ -O3 -std=c++17 -pthread -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
|
||||
Try :
|
||||
c++ -O3 -std=c++17 -pthread -o amalgamate_demo amalgamate_demo.cpp && ./amalgamate_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
|
||||
|
|
|
@ -3,16 +3,16 @@
|
|||
# Generates an "amalgamation build" for roaring. Inspired by similar
|
||||
# script used by whefs.
|
||||
########################################################################
|
||||
set -e
|
||||
|
||||
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
|
||||
|
||||
echo "We are about to amalgamate all simdjson files into one source file. "
|
||||
echo "See https://www.sqlite.org/amalgamation.html and https://en.wikipedia.org/wiki/Single_Compilation_Unit for rationale. "
|
||||
|
||||
AMAL_H="simdjson.h"
|
||||
AMAL_C="simdjson.cpp"
|
||||
|
||||
SRCPATH="$SCRIPTPATH/src"
|
||||
INCLUDEPATH="$SCRIPTPATH/include"
|
||||
if [ -z "$AMALGAMATE_SOURCE_PATH" ]; then AMALGAMATE_SOURCE_PATH="$SCRIPTPATH/../src"; fi
|
||||
if [ -z "$AMALGAMATE_INCLUDE_PATH" ]; then AMALGAMATE_INCLUDE_PATH="$SCRIPTPATH/../include"; fi
|
||||
if [ -z "$AMALGAMATE_OUTPUT_PATH" ]; then AMALGAMATE_OUTPUT_PATH="$SCRIPTPATH"; fi
|
||||
|
||||
# this list excludes the "src/generic headers"
|
||||
ALLCFILES="
|
||||
|
@ -27,14 +27,14 @@ simdjson.h
|
|||
found_includes=()
|
||||
|
||||
for file in ${ALLCFILES}; do
|
||||
test -e "$SRCPATH/$file" && continue
|
||||
echo "FATAL: source file [$SRCPATH/$file] not found."
|
||||
test -e "$AMALGAMATE_SOURCE_PATH/$file" && continue
|
||||
echo "FATAL: source file [$AMALGAMATE_SOURCE_PATH/$file] not found."
|
||||
exit 127
|
||||
done
|
||||
|
||||
for file in ${ALLCHEADERS}; do
|
||||
test -e "$INCLUDEPATH/$file" && continue
|
||||
echo "FATAL: source file [$INCLUDEPATH/$file] not found."
|
||||
test -e "$AMALGAMATE_INCLUDE_PATH/$file" && continue
|
||||
echo "FATAL: source file [$AMALGAMATE_INCLUDE_PATH/$file] not found."
|
||||
exit 127
|
||||
done
|
||||
|
||||
|
@ -42,18 +42,18 @@ function doinclude()
|
|||
{
|
||||
file=$1
|
||||
line="${@:2}"
|
||||
if [ -f $INCLUDEPATH/$file ]; then
|
||||
if [ -f $AMALGAMATE_INCLUDE_PATH/$file ]; then
|
||||
if [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
|
||||
found_includes+=("$file")
|
||||
dofile $INCLUDEPATH/$file
|
||||
dofile $AMALGAMATE_INCLUDE_PATH $file
|
||||
fi;
|
||||
elif [ -f $SRCPATH/$file ]; then
|
||||
elif [ -f $AMALGAMATE_SOURCE_PATH/$file ]; then
|
||||
# generic includes are included multiple times
|
||||
if [[ "${file}" == *'generic/'*'.h' ]]; then
|
||||
dofile $SRCPATH/$file
|
||||
dofile $AMALGAMATE_SOURCE_PATH $file
|
||||
elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
|
||||
found_includes+=("$file")
|
||||
dofile $SRCPATH/$file
|
||||
dofile $AMALGAMATE_SOURCE_PATH $file
|
||||
else
|
||||
echo "/* $file already included: $line */"
|
||||
fi
|
||||
|
@ -65,9 +65,9 @@ function doinclude()
|
|||
|
||||
function dofile()
|
||||
{
|
||||
file="$1/$2"
|
||||
# Last lines are always ignored. Files should end by an empty lines.
|
||||
RELFILE=${1#"$SCRIPTPATH/"}
|
||||
echo "/* begin file $RELFILE */"
|
||||
echo "/* begin file ${2} */"
|
||||
# echo "#line 8 \"$1\"" ## redefining the line/file is not nearly as useful as it sounds for debugging. It breaks IDEs.
|
||||
while IFS= read -r line || [ -n "$line" ];
|
||||
do
|
||||
|
@ -84,23 +84,31 @@ function dofile()
|
|||
# Otherwise we simply copy the line
|
||||
echo "$line"
|
||||
fi
|
||||
done < "$1"
|
||||
done < "$file"
|
||||
echo "/* end file $RELFILE */"
|
||||
}
|
||||
|
||||
timestamp=$(date)
|
||||
mkdir -p $AMALGAMATE_OUTPUT_PATH
|
||||
|
||||
AMAL_H="${AMALGAMATE_OUTPUT_PATH}/simdjson.h"
|
||||
AMAL_C="${AMALGAMATE_OUTPUT_PATH}/simdjson.cpp"
|
||||
DEMOCPP="${AMALGAMATE_OUTPUT_PATH}/amalgamate_demo.cpp"
|
||||
README="$AMALGAMATE_OUTPUT_PATH/README.md"
|
||||
|
||||
echo "Creating ${AMAL_H}..."
|
||||
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_H}"
|
||||
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_H}
|
||||
{
|
||||
for h in ${ALLCHEADERS}; do
|
||||
doinclude $h "ERROR $h not found"
|
||||
done
|
||||
} >> "${AMAL_H}"
|
||||
} >> ${AMAL_H}
|
||||
|
||||
|
||||
echo "Creating ${AMAL_C}..."
|
||||
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
|
||||
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_C}
|
||||
{
|
||||
echo "#include \"${AMAL_H}\""
|
||||
echo "#include \"simdjson.h\""
|
||||
|
||||
echo ""
|
||||
echo "/* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */"
|
||||
|
@ -110,14 +118,13 @@ echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
|
|||
echo ""
|
||||
|
||||
for file in ${ALLCFILES}; do
|
||||
dofile "$SRCPATH/$file"
|
||||
dofile $AMALGAMATE_SOURCE_PATH $file
|
||||
done
|
||||
} >> "${AMAL_C}"
|
||||
} >> ${AMAL_C}
|
||||
|
||||
|
||||
DEMOCPP="amalgamation_demo.cpp"
|
||||
echo "Creating ${DEMOCPP}..."
|
||||
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${DEMOCPP}"
|
||||
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${DEMOCPP}
|
||||
cat <<< '
|
||||
#include <iostream>
|
||||
#include "simdjson.h"
|
||||
|
@ -128,11 +135,14 @@ int main(int argc, char *argv[]) {
|
|||
}
|
||||
const char * filename = argv[1];
|
||||
simdjson::dom::parser parser;
|
||||
auto [doc, error] = parser.load(filename); // do the parsing
|
||||
simdjson::error_code error;
|
||||
UNUSED simdjson::dom::element elem;
|
||||
parser.load(filename).tie(elem, error); // do the parsing
|
||||
if (error) {
|
||||
std::cout << "parse failed" << std::endl;
|
||||
std::cout << "error code: " << error << std::endl;
|
||||
std::cout << error << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
} else {
|
||||
std::cout << "parse valid" << std::endl;
|
||||
}
|
||||
|
@ -149,34 +159,31 @@ int main(int argc, char *argv[]) {
|
|||
std::cout << "parse_many failed" << std::endl;
|
||||
std::cout << "error code: " << error << std::endl;
|
||||
std::cout << error << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
} else {
|
||||
std::cout << "parse_many valid" << std::endl;
|
||||
}
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
' >> "${DEMOCPP}"
|
||||
' >> ${DEMOCPP}
|
||||
|
||||
echo "Done with all files generation. "
|
||||
CPPBIN=$(basename ${DEMOCPP} .cpp)
|
||||
|
||||
echo "Files have been written to directory: $PWD "
|
||||
ls -la ${AMAL_C} ${AMAL_H} ${DEMOCPP}
|
||||
echo "Try :" > ${README}
|
||||
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP##*/} && ./${CPPBIN##*/} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" >> ${README}
|
||||
|
||||
echo "Done with all files generation."
|
||||
|
||||
echo "Files have been written to directory: ${AMALGAMATE_OUTPUT_PATH}/"
|
||||
ls -la ${AMAL_C} ${AMAL_H} ${DEMOCPP} ${README}
|
||||
|
||||
#
|
||||
# Instructions to create demo
|
||||
#
|
||||
echo ""
|
||||
echo "Giving final instructions:"
|
||||
|
||||
|
||||
CPPBIN=${DEMOCPP%%.*}
|
||||
|
||||
echo "Try :"
|
||||
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson"
|
||||
|
||||
SINGLEHDR=$SCRIPTPATH/singleheader
|
||||
echo "Copying files to $SCRIPTPATH/singleheader "
|
||||
mkdir -p $SINGLEHDR
|
||||
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md
|
||||
cp ${AMAL_C} ${AMAL_H} ${DEMOCPP} $SINGLEHDR
|
||||
ls $SINGLEHDR
|
||||
|
||||
cd $SINGLEHDR && c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
|
||||
cat ${README}
|
||||
|
||||
lowercase(){
|
||||
echo "$1" | tr 'A-Z' 'a-z'
|
8
singleheader/amalgamation_demo.cpp → singleheader/amalgamate_demo.cpp
Executable file → Normal file
8
singleheader/amalgamation_demo.cpp → singleheader/amalgamate_demo.cpp
Executable file → Normal file
|
@ -1,4 +1,4 @@
|
|||
/* auto-generated on Sun Apr 19 11:36:20 PDT 2020. Do not edit! */
|
||||
/* auto-generated on Mon Apr 20 11:05:12 PDT 2020. Do not edit! */
|
||||
|
||||
#include <iostream>
|
||||
#include "simdjson.h"
|
||||
|
@ -9,11 +9,14 @@ int main(int argc, char *argv[]) {
|
|||
}
|
||||
const char * filename = argv[1];
|
||||
simdjson::dom::parser parser;
|
||||
auto [doc, error] = parser.load(filename); // do the parsing
|
||||
simdjson::error_code error;
|
||||
UNUSED simdjson::dom::element elem;
|
||||
parser.load(filename).tie(elem, error); // do the parsing
|
||||
if (error) {
|
||||
std::cout << "parse failed" << std::endl;
|
||||
std::cout << "error code: " << error << std::endl;
|
||||
std::cout << error << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
} else {
|
||||
std::cout << "parse valid" << std::endl;
|
||||
}
|
||||
|
@ -30,6 +33,7 @@ int main(int argc, char *argv[]) {
|
|||
std::cout << "parse_many failed" << std::endl;
|
||||
std::cout << "error code: " << error << std::endl;
|
||||
std::cout << error << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
} else {
|
||||
std::cout << "parse_many valid" << std::endl;
|
||||
}
|
File diff suppressed because it is too large
Load Diff
|
@ -1,5 +1,5 @@
|
|||
/* auto-generated on Sun Apr 19 11:36:20 PDT 2020. Do not edit! */
|
||||
/* begin file include/simdjson.h */
|
||||
/* auto-generated on Mon Apr 20 11:05:12 PDT 2020. Do not edit! */
|
||||
/* begin file simdjson.h */
|
||||
#ifndef SIMDJSON_H
|
||||
#define SIMDJSON_H
|
||||
|
||||
|
@ -9,7 +9,7 @@
|
|||
* Check the [README.md](https://github.com/lemire/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second).
|
||||
*/
|
||||
|
||||
/* begin file include/simdjson/compiler_check.h */
|
||||
/* begin file simdjson/compiler_check.h */
|
||||
#ifndef SIMDJSON_COMPILER_CHECK_H
|
||||
#define SIMDJSON_COMPILER_CHECK_H
|
||||
|
||||
|
@ -45,10 +45,10 @@
|
|||
#endif
|
||||
|
||||
#endif // SIMDJSON_COMPILER_CHECK_H
|
||||
/* end file include/simdjson/compiler_check.h */
|
||||
/* end file */
|
||||
|
||||
// Public API
|
||||
/* begin file include/simdjson/simdjson_version.h */
|
||||
/* begin file simdjson/simdjson_version.h */
|
||||
// /include/simdjson/simdjson_version.h automatically generated by release.py,
|
||||
// do not change by hand
|
||||
#ifndef SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
@ -75,17 +75,17 @@ enum {
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_SIMDJSON_VERSION_H
|
||||
/* end file include/simdjson/simdjson_version.h */
|
||||
/* begin file include/simdjson/error.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/error.h */
|
||||
#ifndef SIMDJSON_ERROR_H
|
||||
#define SIMDJSON_ERROR_H
|
||||
|
||||
/* begin file include/simdjson/common_defs.h */
|
||||
/* begin file simdjson/common_defs.h */
|
||||
#ifndef SIMDJSON_COMMON_DEFS_H
|
||||
#define SIMDJSON_COMMON_DEFS_H
|
||||
|
||||
#include <cassert>
|
||||
/* begin file include/simdjson/portability.h */
|
||||
/* begin file simdjson/portability.h */
|
||||
#ifndef SIMDJSON_PORTABILITY_H
|
||||
#define SIMDJSON_PORTABILITY_H
|
||||
|
||||
|
@ -242,7 +242,7 @@ static inline void aligned_free_char(char *mem_block) {
|
|||
}
|
||||
} // namespace simdjson
|
||||
#endif // SIMDJSON_PORTABILITY_H
|
||||
/* end file include/simdjson/portability.h */
|
||||
/* end file */
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
|
@ -367,7 +367,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
|||
#if (!SIMDJSON_CPLUSPLUS17)
|
||||
|
||||
SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
|
||||
/* begin file include/simdjson/nonstd/string_view.hpp */
|
||||
/* begin file simdjson/nonstd/string_view.hpp */
|
||||
// Copyright 2017-2019 by Martin Moene
|
||||
//
|
||||
// string-view lite, a C++17-like string_view for C++98 and later.
|
||||
|
@ -1897,7 +1897,7 @@ nssv_RESTORE_WARNINGS()
|
|||
|
||||
#endif // nssv_HAVE_STD_STRING_VIEW
|
||||
#endif // NONSTD_SV_LITE_H_INCLUDED
|
||||
/* end file include/simdjson/nonstd/string_view.hpp */
|
||||
/* end file */
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
namespace std {
|
||||
|
@ -1906,7 +1906,7 @@ namespace std {
|
|||
#endif // if (SIMDJSON_CPLUSPLUS < 201703L)
|
||||
|
||||
#endif // SIMDJSON_COMMON_DEFS_H
|
||||
/* end file include/simdjson/nonstd/string_view.hpp */
|
||||
/* end file */
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
|
@ -2136,8 +2136,8 @@ inline const std::string &error_message(int error) noexcept;
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_ERROR_H
|
||||
/* end file include/simdjson/nonstd/string_view.hpp */
|
||||
/* begin file include/simdjson/padded_string.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/padded_string.h */
|
||||
#ifndef SIMDJSON_PADDED_STRING_H
|
||||
#define SIMDJSON_PADDED_STRING_H
|
||||
|
||||
|
@ -2277,8 +2277,8 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_PADDED_STRING_H
|
||||
/* end file include/simdjson/padded_string.h */
|
||||
/* begin file include/simdjson/implementation.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/implementation.h */
|
||||
#ifndef SIMDJSON_IMPLEMENTATION_H
|
||||
#define SIMDJSON_IMPLEMENTATION_H
|
||||
|
||||
|
@ -2286,7 +2286,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
|
|||
#include <string>
|
||||
#include <atomic>
|
||||
#include <vector>
|
||||
/* begin file include/simdjson/document.h */
|
||||
/* begin file simdjson/document.h */
|
||||
#ifndef SIMDJSON_DOCUMENT_H
|
||||
#define SIMDJSON_DOCUMENT_H
|
||||
|
||||
|
@ -2295,7 +2295,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
|
|||
#include <string>
|
||||
#include <limits>
|
||||
#include <sstream>
|
||||
/* begin file include/simdjson/simdjson.h */
|
||||
/* begin file simdjson/simdjson.h */
|
||||
/**
|
||||
* @file
|
||||
* @deprecated We'll be removing this file so it isn't confused with the top level simdjson.h
|
||||
|
@ -2305,7 +2305,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept;
|
|||
|
||||
|
||||
#endif // SIMDJSON_H
|
||||
/* end file include/simdjson/simdjson.h */
|
||||
/* end file */
|
||||
|
||||
namespace simdjson {
|
||||
namespace dom {
|
||||
|
@ -3634,7 +3634,7 @@ public:
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_DOCUMENT_H
|
||||
/* end file include/simdjson/simdjson.h */
|
||||
/* end file */
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
|
@ -3864,8 +3864,8 @@ extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> activ
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_IMPLEMENTATION_H
|
||||
/* end file include/simdjson/simdjson.h */
|
||||
/* begin file include/simdjson/document_stream.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/document_stream.h */
|
||||
#ifndef SIMDJSON_DOCUMENT_STREAM_H
|
||||
#define SIMDJSON_DOCUMENT_STREAM_H
|
||||
|
||||
|
@ -4012,16 +4012,16 @@ private:
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_DOCUMENT_STREAM_H
|
||||
/* end file include/simdjson/document_stream.h */
|
||||
/* end file */
|
||||
|
||||
// // Deprecated API
|
||||
/* begin file include/simdjson/jsonparser.h */
|
||||
/* begin file simdjson/jsonparser.h */
|
||||
// TODO Remove this -- deprecated API and files
|
||||
|
||||
#ifndef SIMDJSON_JSONPARSER_H
|
||||
#define SIMDJSON_JSONPARSER_H
|
||||
|
||||
/* begin file include/simdjson/parsedjson.h */
|
||||
/* begin file simdjson/parsedjson.h */
|
||||
// TODO Remove this -- deprecated API and files
|
||||
|
||||
#ifndef SIMDJSON_PARSEDJSON_H
|
||||
|
@ -4037,8 +4037,8 @@ using ParsedJson [[deprecated("Use dom::parser instead")]] = dom::parser;
|
|||
|
||||
} // namespace simdjson
|
||||
#endif
|
||||
/* end file include/simdjson/parsedjson.h */
|
||||
/* begin file include/simdjson/jsonioutil.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/jsonioutil.h */
|
||||
#ifndef SIMDJSON_JSONIOUTIL_H
|
||||
#define SIMDJSON_JSONIOUTIL_H
|
||||
|
||||
|
@ -4064,7 +4064,7 @@ inline padded_string get_corpus(const char *path) {
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_JSONIOUTIL_H
|
||||
/* end file include/simdjson/jsonioutil.h */
|
||||
/* end file */
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
|
@ -4174,8 +4174,8 @@ dom::parser build_parsed_json(const char *buf) noexcept = delete;
|
|||
} // namespace simdjson
|
||||
|
||||
#endif
|
||||
/* end file include/simdjson/jsonioutil.h */
|
||||
/* begin file include/simdjson/parsedjson_iterator.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/parsedjson_iterator.h */
|
||||
// TODO Remove this -- deprecated API and files
|
||||
|
||||
#ifndef SIMDJSON_PARSEDJSON_ITERATOR_H
|
||||
|
@ -4188,7 +4188,7 @@ dom::parser build_parsed_json(const char *buf) noexcept = delete;
|
|||
#include <limits>
|
||||
#include <stdexcept>
|
||||
|
||||
/* begin file include/simdjson/internal/jsonformatutils.h */
|
||||
/* begin file simdjson/internal/jsonformatutils.h */
|
||||
#ifndef SIMDJSON_INTERNAL_JSONFORMATUTILS_H
|
||||
#define SIMDJSON_INTERNAL_JSONFORMATUTILS_H
|
||||
|
||||
|
@ -4254,7 +4254,7 @@ inline std::ostream& operator<<(std::ostream& out, const escape_json_string &une
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INTERNAL_JSONFORMATUTILS_H
|
||||
/* end file include/simdjson/internal/jsonformatutils.h */
|
||||
/* end file */
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
|
@ -4507,10 +4507,10 @@ public:
|
|||
} // namespace simdjson
|
||||
|
||||
#endif
|
||||
/* end file include/simdjson/internal/jsonformatutils.h */
|
||||
/* end file */
|
||||
|
||||
// // Inline functions
|
||||
/* begin file include/simdjson/inline/document.h */
|
||||
/* begin file simdjson/inline/document.h */
|
||||
#ifndef SIMDJSON_INLINE_DOCUMENT_H
|
||||
#define SIMDJSON_INLINE_DOCUMENT_H
|
||||
|
||||
|
@ -5679,8 +5679,8 @@ inline std::string_view internal::tape_ref::get_string_view() const noexcept {
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_DOCUMENT_H
|
||||
/* end file include/simdjson/inline/document.h */
|
||||
/* begin file include/simdjson/inline/document_stream.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/inline/document_stream.h */
|
||||
#ifndef SIMDJSON_INLINE_DOCUMENT_STREAM_H
|
||||
#define SIMDJSON_INLINE_DOCUMENT_STREAM_H
|
||||
|
||||
|
@ -5961,8 +5961,8 @@ inline error_code document_stream::json_parse() noexcept {
|
|||
} // namespace dom
|
||||
} // namespace simdjson
|
||||
#endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H
|
||||
/* end file include/simdjson/inline/document_stream.h */
|
||||
/* begin file include/simdjson/inline/error.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/inline/error.h */
|
||||
#ifndef SIMDJSON_INLINE_ERROR_H
|
||||
#define SIMDJSON_INLINE_ERROR_H
|
||||
|
||||
|
@ -6101,8 +6101,8 @@ really_inline simdjson_result<T>::simdjson_result() noexcept
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_ERROR_H
|
||||
/* end file include/simdjson/inline/error.h */
|
||||
/* begin file include/simdjson/inline/padded_string.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/inline/padded_string.h */
|
||||
#ifndef SIMDJSON_INLINE_PADDED_STRING_H
|
||||
#define SIMDJSON_INLINE_PADDED_STRING_H
|
||||
|
||||
|
@ -6242,8 +6242,8 @@ inline simdjson_result<padded_string> padded_string::load(const std::string &fil
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_PADDED_STRING_H
|
||||
/* end file include/simdjson/inline/padded_string.h */
|
||||
/* begin file include/simdjson/inline/parsedjson_iterator.h */
|
||||
/* end file */
|
||||
/* begin file simdjson/inline/parsedjson_iterator.h */
|
||||
#ifndef SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
|
||||
#define SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
|
||||
|
||||
|
@ -6717,7 +6717,7 @@ bool dom::parser::Iterator::relative_move_to(const char *pointer,
|
|||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
|
||||
/* end file include/simdjson/inline/parsedjson_iterator.h */
|
||||
/* end file */
|
||||
|
||||
#endif // SIMDJSON_H
|
||||
/* end file include/simdjson/inline/parsedjson_iterator.h */
|
||||
/* end file */
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
# target_link_libraries(my-program simdjson-include-source) gives you the header and source
|
||||
# directories. It does not specify any compiler flags.
|
||||
#
|
||||
|
||||
add_library(simdjson-include-source INTERFACE)
|
||||
target_link_libraries(simdjson-include-source INTERFACE simdjson-headers)
|
||||
target_include_directories(simdjson-include-source INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
|
||||
|
|
|
@ -50,25 +50,18 @@ really_inline int count_ones(uint64_t input_num) {
|
|||
|
||||
really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
|
||||
#ifdef _MSC_VER
|
||||
// todo: this might fail under visual studio for ARM
|
||||
return _addcarry_u64(0, value1, value2,
|
||||
reinterpret_cast<unsigned __int64 *>(result));
|
||||
*result = value1 + value2;
|
||||
return *result < value1;
|
||||
#else
|
||||
return __builtin_uaddll_overflow(value1, value2,
|
||||
(unsigned long long *)result);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma intrinsic(_umul128) // todo: this might fail under visual studio for ARM
|
||||
#endif
|
||||
|
||||
really_inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
|
||||
#ifdef _MSC_VER
|
||||
// todo: this might fail under visual studio for ARM
|
||||
uint64_t high;
|
||||
*result = _umul128(value1, value2, &high);
|
||||
return high;
|
||||
*result = value1 * value2;
|
||||
return !!__umulh(value1, value2);
|
||||
#else
|
||||
return __builtin_umulll_overflow(value1, value2, (unsigned long long *)result);
|
||||
#endif
|
||||
|
|
|
@ -26,7 +26,7 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
|
|||
memcpy(&val, chars, sizeof(uint64_t));
|
||||
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
|
||||
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
|
||||
return (val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32;
|
||||
return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
|
||||
}
|
||||
|
||||
#define SWAR_NUMBER_PARSING
|
||||
|
|
|
@ -50,7 +50,7 @@ namespace simd {
|
|||
typedef uint16_t bitmask_t;
|
||||
typedef uint32_t bitmask2_t;
|
||||
|
||||
static really_inline simd8<bool> splat(bool _value) { return vmovq_n_u8(-(!!_value)); }
|
||||
static really_inline simd8<bool> splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); }
|
||||
|
||||
really_inline simd8(const uint8x16_t _value) : base_u8<bool>(_value) {}
|
||||
// False constructor
|
||||
|
@ -159,8 +159,8 @@ namespace simd {
|
|||
really_inline void compress(uint16_t mask, L * output) const {
|
||||
// this particular implementation was inspired by work done by @animetosho
|
||||
// we do it in two steps, first 8 bytes and then second 8 bytes
|
||||
uint8_t mask1 = static_cast<uint8_t>(mask); // least significant 8 bits
|
||||
uint8_t mask2 = static_cast<uint8_t>(mask >> 8); // most significant 8 bits
|
||||
uint8_t mask1 = uint8_t(mask); // least significant 8 bits
|
||||
uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits
|
||||
// next line just loads the 64-bit values thintable_epi8[mask1] and
|
||||
// thintable_epi8[mask2] into a 128-bit register, using only
|
||||
// two instructions on most compilers.
|
||||
|
@ -309,10 +309,10 @@ namespace simd {
|
|||
}
|
||||
|
||||
really_inline void compress(uint64_t mask, T * output) const {
|
||||
this->chunks[0].compress(mask, output);
|
||||
this->chunks[1].compress(mask >> 16, output + 16 - count_ones(mask & 0xFFFF));
|
||||
this->chunks[2].compress(mask >> 32, output + 32 - count_ones(mask & 0xFFFFFFFF));
|
||||
this->chunks[3].compress(mask >> 48, output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
|
||||
this->chunks[0].compress(uint16_t(mask), output);
|
||||
this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF));
|
||||
this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF));
|
||||
this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
|
|
|
@ -40,8 +40,8 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8
|
|||
// smash them together into a 64-byte mask and get the bitmask from there.
|
||||
uint64_t bs_and_quote = simd8x64<bool>(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask();
|
||||
return {
|
||||
static_cast<uint32_t>(bs_and_quote), // bs_bits
|
||||
static_cast<uint32_t>(bs_and_quote >> 32) // quote_bits
|
||||
uint32_t(bs_and_quote), // bs_bits
|
||||
uint32_t(bs_and_quote >> 32) // quote_bits
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
@ -94,7 +94,7 @@ really_inline uint8_t *parser::on_start_string() noexcept {
|
|||
}
|
||||
|
||||
really_inline bool parser::on_end_string(uint8_t *dst) noexcept {
|
||||
uint32_t str_length = dst - (current_string_buf_loc + sizeof(uint32_t));
|
||||
uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t)));
|
||||
// TODO check for overflow in case someone has a crazy string (>=4GB?)
|
||||
// But only add the overflow check when the document itself exceeds 4GB
|
||||
// Currently unneeded because we refuse to parse docs larger or equal to 4GB.
|
||||
|
@ -126,7 +126,7 @@ really_inline bool parser::on_number_double(double value) noexcept {
|
|||
}
|
||||
|
||||
really_inline void parser::write_tape(uint64_t val, internal::tape_type t) noexcept {
|
||||
doc.tape[current_loc++] = val | ((static_cast<uint64_t>(static_cast<char>(t))) << 56);
|
||||
doc.tape[current_loc++] = val | ((uint64_t(char(t))) << 56);
|
||||
}
|
||||
|
||||
// this function is responsible for annotating the start of the scope
|
||||
|
@ -136,7 +136,7 @@ really_inline void parser::end_scope(uint32_t depth) noexcept {
|
|||
// the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff).
|
||||
const uint32_t cntsat = d.count > 0xFFFFFF ? 0xFFFFFF : d.count;
|
||||
// This is a load and an OR. It would be possible to just write once at doc.tape[d.tape_index]
|
||||
doc.tape[d.tape_index] |= current_loc | (static_cast<uint64_t>(cntsat) << 32);
|
||||
doc.tape[d.tape_index] |= current_loc | (uint64_t(cntsat) << 32);
|
||||
}
|
||||
|
||||
} // namespace simdjson
|
||||
|
|
|
@ -7,6 +7,23 @@
|
|||
namespace simdjson {
|
||||
namespace fallback {
|
||||
|
||||
#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64)
|
||||
static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) {
|
||||
unsigned long x0 = (unsigned long)x, top, bottom;
|
||||
_BitScanForward(&top, (unsigned long)(x >> 32));
|
||||
_BitScanForward(&bottom, x0);
|
||||
*ret = x0 ? bottom : 32 + top;
|
||||
return x != 0;
|
||||
}
|
||||
static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) {
|
||||
unsigned long x1 = (unsigned long)(x >> 32), top, bottom;
|
||||
_BitScanReverse(&top, x1);
|
||||
_BitScanReverse(&bottom, (unsigned long)x);
|
||||
*ret = x1 ? top + 32 : bottom;
|
||||
return x != 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
|
|
|
@ -127,7 +127,7 @@ really_inline error_code scan() {
|
|||
}
|
||||
*next_structural_index = len;
|
||||
next_structural_index++;
|
||||
doc_parser.n_structural_indexes = next_structural_index - doc_parser.structural_indexes.get();
|
||||
doc_parser.n_structural_indexes = uint32_t(next_structural_index - doc_parser.structural_indexes.get());
|
||||
return error;
|
||||
}
|
||||
|
||||
|
@ -148,7 +148,7 @@ WARN_UNUSED error_code implementation::stage1(const uint8_t *buf, size_t len, pa
|
|||
if (unlikely(len > parser.capacity())) {
|
||||
return CAPACITY;
|
||||
}
|
||||
stage1::structural_scanner scanner(buf, len, parser, streaming);
|
||||
stage1::structural_scanner scanner(buf, uint32_t(len), parser, streaming);
|
||||
return scanner.scan();
|
||||
}
|
||||
|
||||
|
@ -201,7 +201,7 @@ WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, ui
|
|||
pos += meta[2] | quote;
|
||||
|
||||
i += 1;
|
||||
nonescape = (~nonescape) | (meta[1]);
|
||||
nonescape = uint8_t(~nonescape) | (meta[1]);
|
||||
}
|
||||
dst_len = pos; // we intentionally do not work with a reference
|
||||
// for fear of aliasing
|
||||
|
|
|
@ -11,12 +11,14 @@ public:
|
|||
static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept;
|
||||
|
||||
private:
|
||||
really_inline json_minifier(uint8_t *_dst) : dst{_dst} {}
|
||||
really_inline json_minifier(uint8_t *_dst)
|
||||
: dst{_dst}
|
||||
{}
|
||||
template<size_t STEP_SIZE>
|
||||
really_inline void step(const uint8_t *block_buf, buf_block_reader<STEP_SIZE> &reader) noexcept;
|
||||
really_inline void next(simd::simd8x64<uint8_t> in, json_block block);
|
||||
really_inline error_code finish(uint8_t *dst_start, size_t &dst_len);
|
||||
json_scanner scanner;
|
||||
json_scanner scanner{};
|
||||
uint8_t *dst;
|
||||
};
|
||||
|
||||
|
@ -70,4 +72,4 @@ error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, s
|
|||
return minifier.finish(dst, dst_len);
|
||||
}
|
||||
|
||||
} // namespace stage1
|
||||
} // namespace stage1
|
||||
|
|
|
@ -48,6 +48,7 @@ private:
|
|||
*/
|
||||
class json_scanner {
|
||||
public:
|
||||
json_scanner() {}
|
||||
really_inline json_block next(const simd::simd8x64<uint8_t> in);
|
||||
really_inline error_code finish(bool streaming);
|
||||
|
||||
|
@ -55,7 +56,7 @@ private:
|
|||
// Whether the last character of the previous iteration is part of a scalar token
|
||||
// (anything except whitespace or an operator).
|
||||
uint64_t prev_scalar = 0ULL;
|
||||
json_string_scanner string_scanner;
|
||||
json_string_scanner string_scanner{};
|
||||
};
|
||||
|
||||
|
||||
|
@ -74,7 +75,7 @@ really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) {
|
|||
|
||||
//
|
||||
// Check if the current character follows a matching character, with possible "filler" between.
|
||||
// For example, this checks for empty curly braces, e.g.
|
||||
// For example, this checks for empty curly braces, e.g.
|
||||
//
|
||||
// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { <whitespace>* }
|
||||
//
|
||||
|
@ -100,4 +101,4 @@ really_inline error_code json_scanner::finish(bool streaming) {
|
|||
return string_scanner.finish(streaming);
|
||||
}
|
||||
|
||||
} // namespace stage1
|
||||
} // namespace stage1
|
||||
|
|
|
@ -107,7 +107,7 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64<u
|
|||
const uint64_t in_string = prefix_xor(quote) ^ prev_in_string;
|
||||
// right shift of a signed value expected to be well-defined and standard
|
||||
// compliant as of C++20, John Regher from Utah U. says this is fine code
|
||||
prev_in_string = static_cast<uint64_t>(static_cast<int64_t>(in_string) >> 63);
|
||||
prev_in_string = uint64_t(static_cast<int64_t>(in_string) >> 63);
|
||||
// Use ^ to turn the beginning quote off, and the end quote on.
|
||||
return {
|
||||
backslash,
|
||||
|
|
|
@ -22,7 +22,7 @@ public:
|
|||
// it helps tremendously.
|
||||
if (bits == 0)
|
||||
return;
|
||||
uint32_t cnt = count_ones(bits);
|
||||
int cnt = static_cast<int>(count_ones(bits));
|
||||
|
||||
// Do the first 8 all together
|
||||
for (int i=0; i<8; i++) {
|
||||
|
@ -42,7 +42,7 @@ public:
|
|||
// branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :)
|
||||
// or the start of a value ("abc" true 123) every four characters.
|
||||
if (unlikely(cnt > 16)) {
|
||||
uint32_t i = 16;
|
||||
int i = 16;
|
||||
do {
|
||||
this->tail[i] = idx + trailing_zeroes(bits);
|
||||
bits = clear_lowest_bit(bits);
|
||||
|
@ -61,13 +61,14 @@ public:
|
|||
static error_code index(const uint8_t *buf, size_t len, parser &parser, bool streaming) noexcept;
|
||||
|
||||
private:
|
||||
really_inline json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {}
|
||||
really_inline json_structural_indexer(uint32_t *structural_indexes)
|
||||
: indexer{structural_indexes} {}
|
||||
template<size_t STEP_SIZE>
|
||||
really_inline void step(const uint8_t *block, buf_block_reader<STEP_SIZE> &reader) noexcept;
|
||||
really_inline void next(simd::simd8x64<uint8_t> in, json_block block, size_t idx);
|
||||
really_inline error_code finish(parser &parser, size_t idx, size_t len, bool streaming);
|
||||
|
||||
json_scanner scanner;
|
||||
json_scanner scanner{};
|
||||
utf8_checker checker{};
|
||||
bit_indexer indexer;
|
||||
uint64_t prev_structurals = 0;
|
||||
|
@ -77,14 +78,14 @@ private:
|
|||
really_inline void json_structural_indexer::next(simd::simd8x64<uint8_t> in, json_block block, size_t idx) {
|
||||
uint64_t unescaped = in.lteq(0x1F);
|
||||
checker.check_next_input(in);
|
||||
indexer.write(idx-64, prev_structurals); // Output *last* iteration's structurals to the parser
|
||||
indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser
|
||||
prev_structurals = block.structural_start();
|
||||
unescaped_chars_error |= block.non_quote_inside_string(unescaped);
|
||||
}
|
||||
|
||||
really_inline error_code json_structural_indexer::finish(parser &parser, size_t idx, size_t len, bool streaming) {
|
||||
// Write out the final iteration's structurals
|
||||
indexer.write(idx-64, prev_structurals);
|
||||
indexer.write(uint32_t(idx-64), prev_structurals);
|
||||
|
||||
error_code error = scanner.finish(streaming);
|
||||
if (unlikely(error != SUCCESS)) { return error; }
|
||||
|
@ -93,7 +94,7 @@ really_inline error_code json_structural_indexer::finish(parser &parser, size_t
|
|||
return UNESCAPED_CHARS;
|
||||
}
|
||||
|
||||
parser.n_structural_indexes = indexer.tail - parser.structural_indexes.get();
|
||||
parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get());
|
||||
/* a valid JSON file cannot have zero structural indexes - we should have
|
||||
* found something */
|
||||
if (unlikely(parser.n_structural_indexes == 0u)) {
|
||||
|
@ -105,7 +106,7 @@ really_inline error_code json_structural_indexer::finish(parser &parser, size_t
|
|||
if (len != parser.structural_indexes[parser.n_structural_indexes - 1]) {
|
||||
/* the string might not be NULL terminated, but we add a virtual NULL
|
||||
* ending character. */
|
||||
parser.structural_indexes[parser.n_structural_indexes++] = len;
|
||||
parser.structural_indexes[parser.n_structural_indexes++] = uint32_t(len);
|
||||
}
|
||||
/* make it safe to dereference one beyond this array */
|
||||
parser.structural_indexes[parser.n_structural_indexes] = 0;
|
||||
|
@ -145,7 +146,7 @@ really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_b
|
|||
// they can make a lot of progress before they need that information.
|
||||
// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that
|
||||
// to finish: utf-8 checks and generating the output from the last iteration.
|
||||
//
|
||||
//
|
||||
// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all
|
||||
// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough
|
||||
// workout.
|
||||
|
@ -172,4 +173,4 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, parser
|
|||
return indexer.finish(parser, reader.block_index(), len, streaming);
|
||||
}
|
||||
|
||||
} // namespace stage1
|
||||
} // namespace stage1
|
||||
|
|
|
@ -16,7 +16,7 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
|
|||
if (-22 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
// convert the integer into a double. This is lossless since
|
||||
// 0 <= i <= 2^53 - 1.
|
||||
double d = i;
|
||||
double d = double(i);
|
||||
//
|
||||
// The general idea is as follows.
|
||||
// If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then
|
||||
|
@ -122,7 +122,7 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
|
|||
///////
|
||||
uint64_t upperbit = upper >> 63;
|
||||
uint64_t mantissa = upper >> (upperbit + 9);
|
||||
lz += 1 ^ upperbit;
|
||||
lz += int(1 ^ upperbit);
|
||||
|
||||
// Here we have mantissa < (1<<54).
|
||||
|
||||
|
@ -276,13 +276,13 @@ never_inline bool parse_large_integer(const uint8_t *const src,
|
|||
++p;
|
||||
i = 0;
|
||||
} else {
|
||||
unsigned char digit = *p - '0';
|
||||
unsigned char digit = static_cast<unsigned char>(*p - '0');
|
||||
i = digit;
|
||||
p++;
|
||||
// the is_made_of_eight_digits_fast routine is unlikely to help here because
|
||||
// we rarely see large integer parts like 123456789
|
||||
while (is_integer(*p)) {
|
||||
digit = *p - '0';
|
||||
digit = static_cast<unsigned char>(*p - '0');
|
||||
if (mul_overflow(i, 10, &i)) {
|
||||
#ifdef JSON_TEST_NUMBERS // for unit testing
|
||||
found_invalid_number(src);
|
||||
|
@ -404,13 +404,13 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
|
|||
#endif
|
||||
return false;
|
||||
}
|
||||
unsigned char digit = *p - '0';
|
||||
unsigned char digit = static_cast<unsigned char>(*p - '0');
|
||||
i = digit;
|
||||
p++;
|
||||
// the is_made_of_eight_digits_fast routine is unlikely to help here because
|
||||
// we rarely see large integer parts like 123456789
|
||||
while (is_integer(*p)) {
|
||||
digit = *p - '0';
|
||||
digit = static_cast<unsigned char>(*p - '0');
|
||||
// a multiplication by 10 is cheaper than an arbitrary integer
|
||||
// multiplication
|
||||
i = 10 * i + digit; // might overflow, we will handle the overflow later
|
||||
|
@ -428,7 +428,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
|
|||
++p;
|
||||
const char *const first_after_period = p;
|
||||
if (is_integer(*p)) {
|
||||
unsigned char digit = *p - '0';
|
||||
unsigned char digit = static_cast<unsigned char>(*p - '0');
|
||||
++p;
|
||||
i = i * 10 + digit; // might overflow + multiplication by 10 is likely
|
||||
// cheaper than arbitrary mult.
|
||||
|
@ -448,7 +448,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
|
|||
}
|
||||
#endif
|
||||
while (is_integer(*p)) {
|
||||
unsigned char digit = *p - '0';
|
||||
unsigned char digit = static_cast<unsigned char>(*p - '0');
|
||||
++p;
|
||||
i = i * 10 + digit; // in rare cases, this will overflow, but that's ok
|
||||
// because we have parse_highprecision_float later.
|
||||
|
@ -456,7 +456,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
|
|||
exponent = first_after_period - p;
|
||||
}
|
||||
int digit_count =
|
||||
p - start_digits - 1; // used later to guard against overflows
|
||||
int(p - start_digits) - 1; // used later to guard against overflows
|
||||
int64_t exp_number = 0; // exponential part
|
||||
if (('e' == *p) || ('E' == *p)) {
|
||||
is_float = true;
|
||||
|
@ -474,16 +474,16 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
|
|||
#endif
|
||||
return false;
|
||||
}
|
||||
unsigned char digit = *p - '0';
|
||||
unsigned char digit = static_cast<unsigned char>(*p - '0');
|
||||
exp_number = digit;
|
||||
p++;
|
||||
if (is_integer(*p)) {
|
||||
digit = *p - '0';
|
||||
digit = static_cast<unsigned char>(*p - '0');
|
||||
exp_number = 10 * exp_number + digit;
|
||||
++p;
|
||||
}
|
||||
if (is_integer(*p)) {
|
||||
digit = *p - '0';
|
||||
digit = static_cast<unsigned char>(*p - '0');
|
||||
exp_number = 10 * exp_number + digit;
|
||||
++p;
|
||||
}
|
||||
|
@ -495,7 +495,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
|
|||
#endif
|
||||
return false;
|
||||
}
|
||||
digit = *p - '0';
|
||||
digit = static_cast<unsigned char>(*p - '0');
|
||||
exp_number = 10 * exp_number + digit;
|
||||
++p;
|
||||
}
|
||||
|
@ -513,7 +513,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
|
|||
start++;
|
||||
}
|
||||
// we over-decrement by one when there is a '.'
|
||||
digit_count -= (start - start_digits);
|
||||
digit_count -= int(start - start_digits);
|
||||
if (digit_count >= 19) {
|
||||
// Ok, chances are good that we had an overflow!
|
||||
// this is almost never going to get called!!!
|
||||
|
|
|
@ -50,7 +50,11 @@ struct unified_machine_addresses {
|
|||
class structural_iterator {
|
||||
public:
|
||||
really_inline structural_iterator(const uint8_t* _buf, size_t _len, const uint32_t *_structural_indexes, size_t next_structural_index)
|
||||
: buf{_buf}, len{_len}, structural_indexes{_structural_indexes}, next_structural{next_structural_index} {}
|
||||
: buf{_buf},
|
||||
len{_len},
|
||||
structural_indexes{_structural_indexes},
|
||||
next_structural{next_structural_index}
|
||||
{}
|
||||
really_inline char advance_char() {
|
||||
idx = structural_indexes[next_structural];
|
||||
next_structural++;
|
||||
|
@ -105,8 +109,8 @@ public:
|
|||
const size_t len;
|
||||
const uint32_t* const structural_indexes;
|
||||
size_t next_structural; // next structural index
|
||||
size_t idx; // location of the structural character in the input (buf)
|
||||
uint8_t c; // used to track the (structural) character we are looking at
|
||||
size_t idx{0}; // location of the structural character in the input (buf)
|
||||
uint8_t c{0}; // used to track the (structural) character we are looking at
|
||||
};
|
||||
|
||||
struct structural_parser {
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
namespace stage2 {
|
||||
|
||||
struct streaming_structural_parser: structural_parser {
|
||||
really_inline streaming_structural_parser(const uint8_t *_buf, size_t _len, parser &_doc_parser, size_t _i) : structural_parser(_buf, _len, _doc_parser, _i) {}
|
||||
really_inline streaming_structural_parser(const uint8_t *buf, size_t len, parser &_doc_parser, uint32_t next_structural) : structural_parser(buf, len, _doc_parser, next_structural) {}
|
||||
|
||||
// override to add streaming
|
||||
WARN_UNUSED really_inline error_code start(UNUSED size_t len, ret_address finish_parser) {
|
||||
|
@ -41,7 +41,7 @@ struct streaming_structural_parser: structural_parser {
|
|||
***********/
|
||||
WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, parser &doc_parser, size_t &next_json) const noexcept {
|
||||
static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES();
|
||||
stage2::streaming_structural_parser parser(buf, len, doc_parser, next_json);
|
||||
stage2::streaming_structural_parser parser(buf, len, doc_parser, uint32_t(next_json));
|
||||
error_code result = parser.start(len, addresses.finish);
|
||||
if (result) { return result; }
|
||||
//
|
||||
|
|
|
@ -33,17 +33,19 @@ really_inline uint64_t clear_lowest_bit(uint64_t input_num) {
|
|||
|
||||
/* result might be undefined when input_num is zero */
|
||||
really_inline int leading_zeroes(uint64_t input_num) {
|
||||
return static_cast<int>(_lzcnt_u64(input_num));
|
||||
return int(_lzcnt_u64(input_num));
|
||||
}
|
||||
|
||||
really_inline int count_ones(uint64_t input_num) {
|
||||
#ifdef _MSC_VER
|
||||
really_inline unsigned __int64 count_ones(uint64_t input_num) {
|
||||
// note: we do not support legacy 32-bit Windows
|
||||
return __popcnt64(input_num);// Visual Studio wants two underscores
|
||||
#else
|
||||
return _popcnt64(input_num);
|
||||
#endif
|
||||
}
|
||||
#else
|
||||
really_inline long long int count_ones(uint64_t input_num) {
|
||||
return _popcnt64(input_num);
|
||||
}
|
||||
#endif
|
||||
|
||||
really_inline bool add_overflow(uint64_t value1, uint64_t value2,
|
||||
uint64_t *result) {
|
||||
|
|
|
@ -61,7 +61,7 @@ namespace simd {
|
|||
// SIMD byte mask type (returned by things like eq and gt)
|
||||
template<>
|
||||
struct simd8<bool>: base8<bool> {
|
||||
static really_inline simd8<bool> splat(bool _value) { return _mm256_set1_epi8(-(!!_value)); }
|
||||
static really_inline simd8<bool> splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); }
|
||||
|
||||
really_inline simd8<bool>() : base8() {}
|
||||
really_inline simd8<bool>(const __m256i _value) : base8<bool>(_value) {}
|
||||
|
@ -125,10 +125,10 @@ namespace simd {
|
|||
really_inline void compress(uint32_t mask, L * output) const {
|
||||
// this particular implementation was inspired by work done by @animetosho
|
||||
// we do it in four steps, first 8 bytes and then second 8 bytes...
|
||||
uint8_t mask1 = static_cast<uint8_t>(mask); // least significant 8 bits
|
||||
uint8_t mask2 = static_cast<uint8_t>(mask >> 8); // second least significant 8 bits
|
||||
uint8_t mask3 = static_cast<uint8_t>(mask >> 16); // ...
|
||||
uint8_t mask4 = static_cast<uint8_t>(mask >> 24); // ...
|
||||
uint8_t mask1 = uint8_t(mask); // least significant 8 bits
|
||||
uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits
|
||||
uint8_t mask3 = uint8_t(mask >> 16); // ...
|
||||
uint8_t mask4 = uint8_t(mask >> 24); // ...
|
||||
// next line just loads the 64-bit values thintable_epi8[mask1] and
|
||||
// thintable_epi8[mask2] into a 128-bit register, using only
|
||||
// two instructions on most compilers.
|
||||
|
@ -305,8 +305,8 @@ namespace simd {
|
|||
}
|
||||
|
||||
really_inline void compress(uint64_t mask, T * output) const {
|
||||
uint32_t mask1 = static_cast<uint32_t>(mask);
|
||||
uint32_t mask2 = static_cast<uint32_t>(mask >> 32);
|
||||
uint32_t mask1 = uint32_t(mask);
|
||||
uint32_t mask2 = uint32_t(mask >> 32);
|
||||
this->chunks[0].compress(mask1, output);
|
||||
this->chunks[1].compress(mask2, output + 32 - count_ones(mask1));
|
||||
}
|
||||
|
@ -347,7 +347,7 @@ namespace simd {
|
|||
}
|
||||
|
||||
really_inline uint64_t to_bitmask() const {
|
||||
uint64_t r_lo = static_cast<uint32_t>(this->chunks[0].to_bitmask());
|
||||
uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask());
|
||||
uint64_t r_hi = this->chunks[1].to_bitmask();
|
||||
return r_lo | (r_hi << 32);
|
||||
}
|
||||
|
|
|
@ -280,27 +280,27 @@ static inline bool is_utf8_continuing(char c) {
|
|||
//
|
||||
inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
|
||||
if (cp <= 0x7F) {
|
||||
c[0] = cp;
|
||||
c[0] = uint8_t(cp);
|
||||
return 1; // ascii
|
||||
}
|
||||
if (cp <= 0x7FF) {
|
||||
c[0] = (cp >> 6) + 192;
|
||||
c[1] = (cp & 63) + 128;
|
||||
c[0] = uint8_t((cp >> 6) + 192);
|
||||
c[1] = uint8_t((cp & 63) + 128);
|
||||
return 2; // universal plane
|
||||
// Surrogates are treated elsewhere...
|
||||
//} //else if (0xd800 <= cp && cp <= 0xdfff) {
|
||||
// return 0; // surrogates // could put assert here
|
||||
} else if (cp <= 0xFFFF) {
|
||||
c[0] = (cp >> 12) + 224;
|
||||
c[1] = ((cp >> 6) & 63) + 128;
|
||||
c[2] = (cp & 63) + 128;
|
||||
c[0] = uint8_t((cp >> 12) + 224);
|
||||
c[1] = uint8_t(((cp >> 6) & 63) + 128);
|
||||
c[2] = uint8_t((cp & 63) + 128);
|
||||
return 3;
|
||||
} else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this
|
||||
// is not needed
|
||||
c[0] = (cp >> 18) + 240;
|
||||
c[1] = ((cp >> 12) & 63) + 128;
|
||||
c[2] = ((cp >> 6) & 63) + 128;
|
||||
c[3] = (cp & 63) + 128;
|
||||
c[0] = uint8_t((cp >> 18) + 240);
|
||||
c[1] = uint8_t(((cp >> 12) & 63) + 128);
|
||||
c[2] = uint8_t(((cp >> 6) & 63) + 128);
|
||||
c[3] = uint8_t((cp & 63) + 128);
|
||||
return 4;
|
||||
}
|
||||
// will return 0 when the code point was too large.
|
||||
|
@ -324,15 +324,31 @@ struct value128 {
|
|||
uint64_t high;
|
||||
};
|
||||
|
||||
#if defined(_MSC_VER) && !defined(_M_X64) // _umul128 for x86, arm, arm64
|
||||
#if defined(_M_ARM)
|
||||
static inline uint64_t __emulu(uint32_t x, uint32_t y) {
|
||||
return x * (uint64_t)y;
|
||||
}
|
||||
#endif
|
||||
static inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) {
|
||||
uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd);
|
||||
uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd);
|
||||
uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32));
|
||||
uint64_t adbc_carry = !!(adbc < ad);
|
||||
uint64_t lo = bd + (adbc << 32);
|
||||
*hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + (adbc_carry << 32) + !!(lo < bd);
|
||||
return lo;
|
||||
}
|
||||
#endif
|
||||
|
||||
really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) {
|
||||
value128 answer;
|
||||
#ifdef _MSC_VER
|
||||
// todo: this might fail under visual studio for ARM
|
||||
answer.low = _umul128(value1, value2, &answer.high);
|
||||
#else
|
||||
__uint128_t r = ((__uint128_t)value1) * value2;
|
||||
answer.low = r;
|
||||
answer.high = r >> 64;
|
||||
answer.low = uint64_t(r);
|
||||
answer.high = uint64_t(r >> 64);
|
||||
#endif
|
||||
return answer;
|
||||
}
|
||||
|
|
|
@ -44,14 +44,16 @@ really_inline int leading_zeroes(uint64_t input_num) {
|
|||
#endif// _MSC_VER
|
||||
}
|
||||
|
||||
really_inline int count_ones(uint64_t input_num) {
|
||||
#ifdef _MSC_VER
|
||||
really_inline unsigned __int64 count_ones(uint64_t input_num) {
|
||||
// note: we do not support legacy 32-bit Windows
|
||||
return __popcnt64(input_num);// Visual Studio wants two underscores
|
||||
#else
|
||||
return _popcnt64(input_num);
|
||||
#endif
|
||||
}
|
||||
#else
|
||||
really_inline long long int count_ones(uint64_t input_num) {
|
||||
return _popcnt64(input_num);
|
||||
}
|
||||
#endif
|
||||
|
||||
really_inline bool add_overflow(uint64_t value1, uint64_t value2,
|
||||
uint64_t *result) {
|
||||
|
|
|
@ -62,7 +62,7 @@ namespace simd {
|
|||
// SIMD byte mask type (returned by things like eq and gt)
|
||||
template<>
|
||||
struct simd8<bool>: base8<bool> {
|
||||
static really_inline simd8<bool> splat(bool _value) { return _mm_set1_epi8(-(!!_value)); }
|
||||
static really_inline simd8<bool> splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); }
|
||||
|
||||
really_inline simd8<bool>() : base8() {}
|
||||
really_inline simd8<bool>(const __m128i _value) : base8<bool>(_value) {}
|
||||
|
@ -124,8 +124,8 @@ namespace simd {
|
|||
really_inline void compress(uint16_t mask, L * output) const {
|
||||
// this particular implementation was inspired by work done by @animetosho
|
||||
// we do it in two steps, first 8 bytes and then second 8 bytes
|
||||
uint8_t mask1 = static_cast<uint8_t>(mask); // least significant 8 bits
|
||||
uint8_t mask2 = static_cast<uint8_t>(mask >> 8); // most significant 8 bits
|
||||
uint8_t mask1 = uint8_t(mask); // least significant 8 bits
|
||||
uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits
|
||||
// next line just loads the 64-bit values thintable_epi8[mask1] and
|
||||
// thintable_epi8[mask2] into a 128-bit register, using only
|
||||
// two instructions on most compilers.
|
||||
|
@ -278,10 +278,10 @@ namespace simd {
|
|||
}
|
||||
|
||||
really_inline void compress(uint64_t mask, T * output) const {
|
||||
this->chunks[0].compress(mask, output);
|
||||
this->chunks[1].compress(mask >> 16, output + 16 - count_ones(mask & 0xFFFF));
|
||||
this->chunks[2].compress(mask >> 32, output + 32 - count_ones(mask & 0xFFFFFFFF));
|
||||
this->chunks[3].compress(mask >> 48, output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
|
||||
this->chunks[0].compress(uint16_t(mask), output);
|
||||
this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF));
|
||||
this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF));
|
||||
this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
|
@ -330,7 +330,7 @@ namespace simd {
|
|||
}
|
||||
|
||||
really_inline uint64_t to_bitmask() const {
|
||||
uint64_t r0 = static_cast<uint32_t>(this->chunks[0].to_bitmask());
|
||||
uint64_t r0 = uint32_t(this->chunks[0].to_bitmask());
|
||||
uint64_t r1 = this->chunks[1].to_bitmask();
|
||||
uint64_t r2 = this->chunks[2].to_bitmask();
|
||||
uint64_t r3 = this->chunks[3].to_bitmask();
|
||||
|
|
|
@ -38,8 +38,8 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8
|
|||
v1.store(dst + 16);
|
||||
uint64_t bs_and_quote = simd8x64<bool>(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask();
|
||||
return {
|
||||
static_cast<uint32_t>(bs_and_quote), // bs_bits
|
||||
static_cast<uint32_t>(bs_and_quote >> 32) // quote_bits
|
||||
uint32_t(bs_and_quote), // bs_bits
|
||||
uint32_t(bs_and_quote >> 32) // quote_bits
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
@ -66,7 +66,7 @@ if (NOT MSVC) # Can't run .sh on windows
|
|||
#
|
||||
add_test(
|
||||
NAME testjson2json
|
||||
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh
|
||||
COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh
|
||||
WORKING_DIRECTORY $<TARGET_FILE_DIR:minify>
|
||||
)
|
||||
set_property(TEST testjson2json APPEND PROPERTY DEPENDS minify json2json)
|
||||
|
@ -79,12 +79,13 @@ if (NOT MSVC) # Can't run .sh on windows
|
|||
add_executable(allparserscheckfile allparserscheckfile.cpp)
|
||||
target_link_libraries(allparserscheckfile competition-all)
|
||||
|
||||
add_test(issue150 ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
|
||||
add_test(issue150 bash ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
|
||||
set_property(TEST issue150 APPEND PROPERTY DEPENDS allparserscheckfile)
|
||||
set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# This should really test whether bash is available.
|
||||
if (NOT MSVC)
|
||||
#
|
||||
# json2json tool test: check that json2json can parse twitter.json
|
||||
|
@ -139,7 +140,6 @@ if(NOT (MSVC AND MSVC_VERSION LESS 1920))
|
|||
# Compile tests that *should fail*
|
||||
add_cpp_test(readme_examples_will_fail_with_exceptions_off WILL_FAIL COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
|
||||
target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0)
|
||||
|
||||
endif()
|
||||
|
||||
|
||||
|
|
|
@ -138,7 +138,7 @@ int main(int argc, char *argv[]) {
|
|||
jsmn_init(&jsmnparser);
|
||||
memcpy(buffer, p.data(), p.size());
|
||||
buffer[p.size()] = '\0';
|
||||
int r = jsmn_parse(&jsmnparser, buffer, p.size(), tokens.get(), p.size());
|
||||
int r = jsmn_parse(&jsmnparser, buffer, p.size(), tokens.get(), static_cast<unsigned int>(p.size()));
|
||||
tokens = nullptr;
|
||||
jsmn_correct = (r > 0);
|
||||
}
|
||||
|
|
|
@ -73,7 +73,7 @@ namespace number_tests {
|
|||
std::cout << __func__ << std::endl;
|
||||
char buf[1024];
|
||||
simdjson::dom::parser parser;
|
||||
int maxulp = 0;
|
||||
uint64_t maxulp = 0;
|
||||
for (int i = -1075; i < 1024; ++i) {// large negative values should be zero.
|
||||
double expected = pow(2, i);
|
||||
auto n = sprintf(buf, "%.*e", std::numeric_limits<double>::max_digits10 - 1, expected);
|
||||
|
@ -81,7 +81,7 @@ namespace number_tests {
|
|||
fflush(NULL);
|
||||
auto [actual, error] = parser.parse(buf, n).get<double>();
|
||||
if (error) { std::cerr << error << std::endl; return false; }
|
||||
int ulp = f64_ulp_dist(actual,expected);
|
||||
uint64_t ulp = f64_ulp_dist(actual,expected);
|
||||
if(ulp > maxulp) maxulp = ulp;
|
||||
if(ulp > 0) {
|
||||
std::cerr << "JSON '" << buf << " parsed to " << actual << " instead of " << expected << std::endl;
|
||||
|
@ -284,12 +284,12 @@ namespace document_tests {
|
|||
data.emplace_back(std::string(buf, n));
|
||||
}
|
||||
for (size_t i = 0; i < n_records; ++i) {
|
||||
auto n = sprintf(buf, "{\"counter\": %f, \"array\": [%s]}", i * 3.1416,
|
||||
auto n = sprintf(buf, "{\"counter\": %f, \"array\": [%s]}", static_cast<double>(i) * 3.1416,
|
||||
(i % 2) ? "true" : "false");
|
||||
data.emplace_back(std::string(buf, n));
|
||||
}
|
||||
for (size_t i = 0; i < n_records; ++i) {
|
||||
auto n = sprintf(buf, "{\"number\": %e}", i * 10000.31321321);
|
||||
auto n = sprintf(buf, "{\"number\": %e}", static_cast<double>(i) * 10000.31321321);
|
||||
data.emplace_back(std::string(buf, n));
|
||||
}
|
||||
data.emplace_back(std::string("true"));
|
||||
|
@ -1563,7 +1563,7 @@ namespace type_tests {
|
|||
&& (expected_value >= 0 ?
|
||||
test_cast<uint64_t>(result, expected_value) :
|
||||
test_cast<uint64_t>(result, NUMBER_OUT_OF_RANGE))
|
||||
&& test_cast<double>(result, expected_value)
|
||||
&& test_cast<double>(result, static_cast<double>(expected_value))
|
||||
&& test_cast<bool>(result, INCORRECT_TYPE)
|
||||
&& test_is_null(result, false);
|
||||
}
|
||||
|
@ -1582,7 +1582,7 @@ namespace type_tests {
|
|||
&& test_cast<const char *>(result, INCORRECT_TYPE)
|
||||
&& test_cast<int64_t>(result, NUMBER_OUT_OF_RANGE)
|
||||
&& test_cast<uint64_t>(result, expected_value)
|
||||
&& test_cast<double>(result, expected_value)
|
||||
&& test_cast<double>(result, static_cast<double>(expected_value))
|
||||
&& test_cast<bool>(result, INCORRECT_TYPE)
|
||||
&& test_is_null(result, false);
|
||||
}
|
||||
|
|
|
@ -31,11 +31,11 @@ static inline bool read_hex(const char *p, unsigned &u) {
|
|||
while (i--) {
|
||||
unsigned char c = *p++;
|
||||
if (c >= '0' && c <= '9') {
|
||||
c -= '0';
|
||||
c = static_cast<unsigned char>(c - '0');
|
||||
} else if (c >= 'a' && c <= 'f') {
|
||||
c = c - 'a' + 10;
|
||||
c = static_cast<unsigned char>(c - 'a' + 10);
|
||||
} else if (c >= 'A' && c <= 'F') {
|
||||
c = c - 'A' + 10;
|
||||
c = static_cast<unsigned char>(c - 'A' + 10);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
@ -48,20 +48,20 @@ static inline bool read_hex(const char *p, unsigned &u) {
|
|||
|
||||
static inline void write_utf8(unsigned codepoint, char *&end) {
|
||||
if (codepoint < 0x80) {
|
||||
*end++ = codepoint;
|
||||
*end++ = static_cast<char>(codepoint);
|
||||
} else if (codepoint < 0x800) {
|
||||
*end++ = 0xC0 | (codepoint >> 6);
|
||||
*end++ = 0x80 | (codepoint & 0x3F);
|
||||
*end++ = static_cast<char>(0xC0 | (codepoint >> 6));
|
||||
*end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
|
||||
} else if (codepoint < 0x10000) {
|
||||
*end++ = 0xE0 | (codepoint >> 12);
|
||||
*end++ = 0x80 | ((codepoint >> 6) & 0x3F);
|
||||
*end++ = 0x80 | (codepoint & 0x3F);
|
||||
*end++ = static_cast<char>(0xE0 | (codepoint >> 12));
|
||||
*end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
|
||||
*end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
|
||||
} else {
|
||||
assert(codepoint < 0x200000);
|
||||
*end++ = 0xF0 | (codepoint >> 18);
|
||||
*end++ = 0x80 | ((codepoint >> 12) & 0x3F);
|
||||
*end++ = 0x80 | ((codepoint >> 6) & 0x3F);
|
||||
*end++ = 0x80 | (codepoint & 0x3F);
|
||||
*end++ = static_cast<char>(0xF0 | (codepoint >> 18));
|
||||
*end++ = static_cast<char>(0x80 | ((codepoint >> 12) & 0x3F));
|
||||
*end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
|
||||
*end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -357,7 +357,7 @@ bool validate(const char *dirname) {
|
|||
"\taverage string length: %.1f \n",
|
||||
name, isok ? " is valid " : " is not valid ", bad_string,
|
||||
good_string, empty_string,
|
||||
(double)total_string_length / good_string);
|
||||
static_cast<double>(total_string_length) / static_cast<double>(good_string));
|
||||
} else if (bad_string > 0) {
|
||||
printf("File %40s %s --- bad strings: %10zu \n", name,
|
||||
isok ? " is valid " : " is not valid ", bad_string);
|
||||
|
|
|
@ -159,5 +159,5 @@ print("modified "+doxyfile+", a backup was made")
|
|||
scriptlocation = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
print("Please run the tests before issuing a release, do make test && make amalgamate \n")
|
||||
print("Please run the tests before issuing a release, do make test && make amalgamate_test \n")
|
||||
print("to issue release, enter \n git commit -a && git push && git tag -a v"+toversionstring(*newversion)+" -m \"version "+toversionstring(*newversion)+"\" && git push --tags \n")
|
||||
|
|
Loading…
Reference in New Issue