From 5d355f1a8b584519ed39256c3be949ef7f8b0dbd Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 19 Aug 2020 18:12:23 -0400 Subject: [PATCH] release candidate (#1132) --- CMakeLists.txt | 10 +- Doxyfile | 2 +- README.md | 4 +- RELEASES.md | 21 +- doc/basics.md | 4 +- doc/basics_doxygen.md | 2 +- include/simdjson/simdjson_version.h | 6 +- singleheader/simdjson.cpp | 2726 +++++++++++++-------------- singleheader/simdjson.h | 682 ++++--- 9 files changed, 1735 insertions(+), 1722 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a66544b..fadcaef2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,11 +6,11 @@ project(simdjson ) set(PROJECT_VERSION_MAJOR 0) -set(PROJECT_VERSION_MINOR 4) -set(PROJECT_VERSION_PATCH 6) -set(SIMDJSON_SEMANTIC_VERSION "0.4.6" CACHE STRING "simdjson semantic version") -set(SIMDJSON_LIB_VERSION "2.0.0" CACHE STRING "simdjson library version") -set(SIMDJSON_LIB_SOVERSION "2" CACHE STRING "simdjson library soversion") +set(PROJECT_VERSION_MINOR 5) +set(PROJECT_VERSION_PATCH 0) +set(SIMDJSON_SEMANTIC_VERSION "0.5.0" CACHE STRING "simdjson semantic version") +set(SIMDJSON_LIB_VERSION "3.0.0" CACHE STRING "simdjson library version") +set(SIMDJSON_LIB_SOVERSION "3" CACHE STRING "simdjson library soversion") set(SIMDJSON_GITHUB_REPOSITORY https://github.com/simdjson/simdjson) include(GNUInstallDirs) diff --git a/Doxyfile b/Doxyfile index b6af5cd3..235f5741 100644 --- a/Doxyfile +++ b/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = simdjson # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "0.4.6" +PROJECT_NUMBER = "0.5.0" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/README.md b/README.md index 560f0edc..0388e606 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html) ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg) ![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg) -[![][license img]][license] [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.4.0/index.html) +[![][license img]][license] [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.5.0/index.html) simdjson : Parsing gigabytes of JSON per second =============================================== @@ -70,7 +70,7 @@ Usage documentation is available: * [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them. * [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and how you can work with it. -* [API](https://simdjson.org/api/0.4.0/annotated.html) contains the automatically generated API documentation. +* [API](https://simdjson.org/api/0.5.0/annotated.html) contains the automatically generated API documentation. Performance results ------------------- diff --git a/RELEASES.md b/RELEASES.md index 8cd014c3..9c31aeff 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,4 +1,23 @@ -# 0.3 +# 0.5 + +## Highlights + +Performance +* Faster and simpler UTF-8 validation with the lookup4 algorithm https://github.com/simdjson/simdjson/pull/993 +* We improved the performance of simdjson under Visual Studio by about 25%. Users will still get better performance with clang-cl (+30%) but the gap has been reduced. https://github.com/simdjson/simdjson/pull/1031 + +Code usability +* In `parse_many`, when parsing streams of JSON documetns, we give to the users runtime control as to whether threads are used (via the parser.threaded attribute). https://github.com/simdjson/simdjson/issues/925 +* Prefixed public macros to avoid name clashes with other libraries. https://github.com/simdjson/simdjson/issues/1035 +* Better documentation regarding package managers (brew, MSYS2, conan, apt, vcpkg, FreeBSD package manager, etc.). +* Better documentation regarding CMake usage. + +Standards +* We improved standard compliance with respect to both the JSON RFC 8259 and JSON Pointer RFC 6901. We added the at_pointer method to nodes for standard-compliant JSON Pointer queries. The legacy `at(std::string_view)` method remains but is deprecated since it is not standard-compliant as per RFC 6901. +* We removed computed GOTOs without sacrificing performance thus improving the C++ standard compliance (since computed GOTOs are compiler-specific extensions). +* Better support for C++20 https://github.com/simdjson/simdjson/pull/1050 + +# 0.4 ## Highlights diff --git a/doc/basics.md b/doc/basics.md index cc541437..e81e99a9 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -66,7 +66,7 @@ include(FetchContent) FetchContent_Declare( simdjson GIT_REPOSITORY https://github.com/simdjson/simdjson.git - GIT_TAG v0.4.7 + GIT_TAG v0.5.0 GIT_SHALLOW TRUE) set(SIMDJSON_JUST_LIBRARY ON CACHE INTERNAL "") @@ -75,7 +75,7 @@ set(SIMDJSON_BUILD_STATIC ON CACHE INTERNAL "") FetchContent_MakeAvailable(simdjson) ``` -You should replace `GIT_TAG v0.4.7` by the version you need (e.g., `GIT_TAG v0.5.0`). If you omit `GIT_TAG v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, +You should replace `GIT_TAG v0.5.0` by the version you need. If you omit `GIT_TAG v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, Elsewhere in your project, you can declare dependencies on simdjson with lines such as these: diff --git a/doc/basics_doxygen.md b/doc/basics_doxygen.md index 17bf78a4..fbb56680 100644 --- a/doc/basics_doxygen.md +++ b/doc/basics_doxygen.md @@ -56,7 +56,7 @@ set(SIMDJSON_BUILD_STATIC ON CACHE INTERNAL "") FetchContent_MakeAvailable(simdjson) ``` -You should replace `GIT_TAG v0.4.7` by the version you need (e.g., `GIT_TAG v0.5.0`). If you omit `GIT_TAG v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, +You should replace `GIT_TAG v0.5.0` by the version you need. If you omit `GIT_TAG v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, Elsewhere in your project, you can declare dependencies on simdjson with lines such as these: diff --git a/include/simdjson/simdjson_version.h b/include/simdjson/simdjson_version.h index a2f5dc1f..be4d40f2 100644 --- a/include/simdjson/simdjson_version.h +++ b/include/simdjson/simdjson_version.h @@ -4,7 +4,7 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION 0.4.6 +#define SIMDJSON_VERSION 0.5.0 namespace simdjson { enum { @@ -15,11 +15,11 @@ enum { /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 4, + SIMDJSON_VERSION_MINOR = 5, /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 6 + SIMDJSON_VERSION_REVISION = 0 }; } // namespace simdjson diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 37ad94f6..aa7ad828 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Tue Aug 18 17:35:47 PDT 2020. Do not edit! */ +/* auto-generated on Wed 19 Aug 2020 17:23:29 EDT. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" @@ -371,18 +371,18 @@ using namespace simdjson; class implementation final : public simdjson::implementation { public: - really_inline implementation() : simdjson::implementation( + simdjson_really_inline implementation() : simdjson::implementation( "haswell", "Intel/AMD AVX2", instruction_set::AVX2 | instruction_set::PCLMULQDQ | instruction_set::BMI1 | instruction_set::BMI2 ) {} - WARN_UNUSED error_code create_dom_parser_implementation( + SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation( size_t capacity, size_t max_length, std::unique_ptr& dst ) const noexcept final; - WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; + SIMDJSON_WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; + SIMDJSON_WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; }; } // namespace haswell @@ -409,14 +409,14 @@ using namespace simdjson::dom; class implementation final : public simdjson::implementation { public: - really_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", instruction_set::SSE42 | instruction_set::PCLMULQDQ) {} - WARN_UNUSED error_code create_dom_parser_implementation( + simdjson_really_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", instruction_set::SSE42 | instruction_set::PCLMULQDQ) {} + SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation( size_t capacity, size_t max_length, std::unique_ptr& dst ) const noexcept final; - WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; + SIMDJSON_WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; + SIMDJSON_WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; }; } // namespace westmere @@ -442,14 +442,14 @@ using namespace simdjson::dom; class implementation final : public simdjson::implementation { public: - really_inline implementation() : simdjson::implementation("arm64", "ARM NEON", instruction_set::NEON) {} - WARN_UNUSED error_code create_dom_parser_implementation( + simdjson_really_inline implementation() : simdjson::implementation("arm64", "ARM NEON", instruction_set::NEON) {} + SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation( size_t capacity, size_t max_length, std::unique_ptr& dst ) const noexcept final; - WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; + SIMDJSON_WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; + SIMDJSON_WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; }; } // namespace arm64 @@ -475,18 +475,18 @@ using namespace simdjson::dom; class implementation final : public simdjson::implementation { public: - really_inline implementation() : simdjson::implementation( + simdjson_really_inline implementation() : simdjson::implementation( "fallback", "Generic fallback implementation", 0 ) {} - WARN_UNUSED error_code create_dom_parser_implementation( + SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation( size_t capacity, size_t max_length, std::unique_ptr& dst ) const noexcept final; - WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; + SIMDJSON_WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; + SIMDJSON_WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final; }; } // namespace fallback @@ -508,20 +508,20 @@ public: const std::string &name() const noexcept final { return set_best()->name(); } const std::string &description() const noexcept final { return set_best()->description(); } uint32_t required_instruction_sets() const noexcept final { return set_best()->required_instruction_sets(); } - WARN_UNUSED error_code create_dom_parser_implementation( + SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation( size_t capacity, size_t max_length, std::unique_ptr& dst ) const noexcept final { return set_best()->create_dom_parser_implementation(capacity, max_length, dst); } - WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final { + SIMDJSON_WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final { return set_best()->minify(buf, len, dst, dst_len); } - WARN_UNUSED bool validate_utf8(const char * buf, size_t len) const noexcept final override { + SIMDJSON_WARN_UNUSED bool validate_utf8(const char * buf, size_t len) const noexcept final override { return set_best()->validate_utf8(buf, len); } - really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} + simdjson_really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} private: const implementation *set_best() const noexcept; }; @@ -546,17 +546,17 @@ const std::initializer_list available_implementation_poi // So we can return UNSUPPORTED_ARCHITECTURE from the parser when there is no support class unsupported_implementation final : public implementation { public: - WARN_UNUSED error_code create_dom_parser_implementation( + SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation( size_t, size_t, std::unique_ptr& ) const noexcept final { return UNSUPPORTED_ARCHITECTURE; } - WARN_UNUSED error_code minify(const uint8_t *, size_t, uint8_t *, size_t &) const noexcept final override { + SIMDJSON_WARN_UNUSED error_code minify(const uint8_t *, size_t, uint8_t *, size_t &) const noexcept final override { return UNSUPPORTED_ARCHITECTURE; } - WARN_UNUSED bool validate_utf8(const char *, size_t) const noexcept final override { + SIMDJSON_WARN_UNUSED bool validate_utf8(const char *, size_t) const noexcept final override { return false; // Just refuse to validate. Given that we have a fallback implementation // it seems unlikely that unsupported_implementation will ever be used. If it is used, // then it will flag all strings as invalid. The alternative is to return an error_code @@ -613,10 +613,10 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{}; SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton}; -WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept { +SIMDJSON_WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept { return active_implementation->minify((const uint8_t *)buf, len, (uint8_t *)dst, dst_len); } -WARN_UNUSED bool validate_utf8(const char *buf, size_t len) noexcept { +SIMDJSON_WARN_UNUSED bool validate_utf8(const char *buf, size_t len) noexcept { return active_implementation->validate_utf8(buf, len); } @@ -1849,7 +1849,7 @@ namespace arm64 { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. NO_SANITIZE_UNDEFINED -really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_really_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long ret; // Search the mask data from least significant bit (LSB) @@ -1862,12 +1862,12 @@ really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { return input_num & (input_num-1); } /* result might be undefined when input_num is zero */ -really_inline int leading_zeroes(uint64_t input_num) { +simdjson_really_inline int leading_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long leading_zero = 0; // Search the mask data from most significant bit (MSB) @@ -1882,11 +1882,11 @@ really_inline int leading_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -really_inline int count_ones(uint64_t input_num) { +simdjson_really_inline int count_ones(uint64_t input_num) { return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); } -really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { +simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO *result = value1 + value2; return *result < value1; @@ -1913,7 +1913,7 @@ namespace arm64 { // // For example, prefix_xor(00100100) == 00011100 // -really_inline uint64_t prefix_xor(uint64_t bitmask) { +simdjson_really_inline uint64_t prefix_xor(uint64_t bitmask) { ///////////// // We could do this with PMULL, but it is apparently slow. // @@ -1971,7 +1971,7 @@ namespace { * You should not use this function except for compile-time constants: * it is not efficient. */ -really_inline uint8x16_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, +simdjson_really_inline uint8x16_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7, uint8_t x8, uint8_t x9, uint8_t x10, uint8_t x11, uint8_t x12, uint8_t x13, uint8_t x14, uint8_t x15, uint8_t x16) { @@ -2002,7 +2002,7 @@ really_inline uint8x16_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, // We have to do the same work for make_int8x16_t -really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, +simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, int8_t x5, int8_t x6, int8_t x7, int8_t x8, int8_t x9, int8_t x10, int8_t x11, int8_t x12, int8_t x13, int8_t x14, int8_t x15, int8_t x16) { @@ -2048,24 +2048,24 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ static const int SIZE = sizeof(value); // Conversion from/to SIMD register - really_inline base_u8(const uint8x16_t _value) : value(_value) {} - really_inline operator const uint8x16_t&() const { return this->value; } - really_inline operator uint8x16_t&() { return this->value; } + simdjson_really_inline base_u8(const uint8x16_t _value) : value(_value) {} + simdjson_really_inline operator const uint8x16_t&() const { return this->value; } + simdjson_really_inline operator uint8x16_t&() { return this->value; } // Bit operations - really_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } - really_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } - really_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } - really_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } - really_inline simd8 operator~() const { return *this ^ 0xFFu; } - really_inline simd8& operator|=(const simd8 other) { auto this_cast = (simd8*)this; *this_cast = *this_cast | other; return *this_cast; } - really_inline simd8& operator&=(const simd8 other) { auto this_cast = (simd8*)this; *this_cast = *this_cast & other; return *this_cast; } - really_inline simd8& operator^=(const simd8 other) { auto this_cast = (simd8*)this; *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_really_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } + simdjson_really_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } + simdjson_really_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } + simdjson_really_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } + simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_really_inline simd8& operator|=(const simd8 other) { auto this_cast = (simd8*)this; *this_cast = *this_cast | other; return *this_cast; } + simdjson_really_inline simd8& operator&=(const simd8 other) { auto this_cast = (simd8*)this; *this_cast = *this_cast & other; return *this_cast; } + simdjson_really_inline simd8& operator^=(const simd8 other) { auto this_cast = (simd8*)this; *this_cast = *this_cast ^ other; return *this_cast; } - really_inline Mask operator==(const simd8 other) const { return vceqq_u8(*this, other); } + simdjson_really_inline Mask operator==(const simd8 other) const { return vceqq_u8(*this, other); } template - really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { return vextq_u8(prev_chunk, *this, 16 - N); } }; @@ -2076,17 +2076,17 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ typedef uint16_t bitmask_t; typedef uint32_t bitmask2_t; - static really_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } + static simdjson_really_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } - really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} + simdjson_really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} // False constructor - really_inline simd8() : simd8(vdupq_n_u8(0)) {} + simdjson_really_inline simd8() : simd8(vdupq_n_u8(0)) {} // Splat constructor - really_inline simd8(bool _value) : simd8(splat(_value)) {} + simdjson_really_inline simd8(bool _value) : simd8(splat(_value)) {} // We return uint32_t instead of uint16_t because that seems to be more efficient for most // purposes (cutting it down to uint16_t costs performance in some compilers). - really_inline uint32_t to_bitmask() const { + simdjson_really_inline uint32_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const uint8x16_t bit_mask = make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80); @@ -2100,26 +2100,26 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } - really_inline bool any() const { return vmaxvq_u8(*this) != 0; } + simdjson_really_inline bool any() const { return vmaxvq_u8(*this) != 0; } }; // Unsigned bytes template<> struct simd8: base_u8 { - static really_inline uint8x16_t splat(uint8_t _value) { return vmovq_n_u8(_value); } - static really_inline uint8x16_t zero() { return vdupq_n_u8(0); } - static really_inline uint8x16_t load(const uint8_t* values) { return vld1q_u8(values); } + static simdjson_really_inline uint8x16_t splat(uint8_t _value) { return vmovq_n_u8(_value); } + static simdjson_really_inline uint8x16_t zero() { return vdupq_n_u8(0); } + static simdjson_really_inline uint8x16_t load(const uint8_t* values) { return vld1q_u8(values); } - really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} + simdjson_really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} // Zero constructor - really_inline simd8() : simd8(zero()) {} + simdjson_really_inline simd8() : simd8(zero()) {} // Array constructor - really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + simdjson_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} // Splat constructor - really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Member-by-member initialization #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO - really_inline simd8( + simdjson_really_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(make_uint8x16_t( @@ -2127,7 +2127,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ v8, v9, v10,v11,v12,v13,v14,v15 )) {} #else - really_inline simd8( + simdjson_really_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(uint8x16_t{ @@ -2137,7 +2137,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ #endif // Repeat 16 values as many times as necessary (usually for lookup tables) - really_inline static simd8 repeat_16( + simdjson_really_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -2148,44 +2148,44 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ } // Store to array - really_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } + simdjson_really_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } // Saturated math - really_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } - really_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } + simdjson_really_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } + simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } // Addition/subtraction are the same for signed and unsigned - really_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } - really_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } - really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + simdjson_really_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } + simdjson_really_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } + simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } // Order-specific operations - really_inline uint8_t max() const { return vmaxvq_u8(*this); } - really_inline uint8_t min() const { return vminvq_u8(*this); } - really_inline simd8 max(const simd8 other) const { return vmaxq_u8(*this, other); } - really_inline simd8 min(const simd8 other) const { return vminq_u8(*this, other); } - really_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } - really_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } - really_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } - really_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } + simdjson_really_inline uint8_t max() const { return vmaxvq_u8(*this); } + simdjson_really_inline uint8_t min() const { return vminvq_u8(*this); } + simdjson_really_inline simd8 max(const simd8 other) const { return vmaxq_u8(*this, other); } + simdjson_really_inline simd8 min(const simd8 other) const { return vminq_u8(*this, other); } + simdjson_really_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } + simdjson_really_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } + simdjson_really_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - really_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } + simdjson_really_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - really_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } + simdjson_really_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } // Bit-specific operations - really_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - really_inline bool any_bits_set_anywhere() const { return this->max() != 0; } - really_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } + simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } + simdjson_really_inline bool any_bits_set_anywhere() const { return this->max() != 0; } + simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } template - really_inline simd8 shr() const { return vshrq_n_u8(*this, N); } + simdjson_really_inline simd8 shr() const { return vshrq_n_u8(*this, N); } template - really_inline simd8 shl() const { return vshlq_n_u8(*this, N); } + simdjson_really_inline simd8 shl() const { return vshlq_n_u8(*this, N); } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template - really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { return lookup_table.apply_lookup_16_to(*this); } @@ -2198,7 +2198,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ // signature simd8 compress(uint16_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - really_inline void compress(uint16_t mask, L * output) const { + simdjson_really_inline void compress(uint16_t mask, L * output) const { // this particular implementation was inspired by work done by @animetosho // we do it in two steps, first 8 bytes and then second 8 bytes uint8_t mask1 = uint8_t(mask); // least significant 8 bits @@ -2230,7 +2230,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ } template - really_inline simd8 lookup_16( + simdjson_really_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -2244,7 +2244,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ } template - really_inline simd8 apply_lookup_16_to(const simd8 original) { + simdjson_really_inline simd8 apply_lookup_16_to(const simd8 original) { return vqtbl1q_u8(*this, simd8(original)); } }; @@ -2254,24 +2254,24 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ struct simd8 { int8x16_t value; - static really_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } - static really_inline simd8 zero() { return vdupq_n_s8(0); } - static really_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } + static simdjson_really_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } + static simdjson_really_inline simd8 zero() { return vdupq_n_s8(0); } + static simdjson_really_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } // Conversion from/to SIMD register - really_inline simd8(const int8x16_t _value) : value{_value} {} - really_inline operator const int8x16_t&() const { return this->value; } - really_inline operator int8x16_t&() { return this->value; } + simdjson_really_inline simd8(const int8x16_t _value) : value{_value} {} + simdjson_really_inline operator const int8x16_t&() const { return this->value; } + simdjson_really_inline operator int8x16_t&() { return this->value; } // Zero constructor - really_inline simd8() : simd8(zero()) {} + simdjson_really_inline simd8() : simd8(zero()) {} // Splat constructor - really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - really_inline simd8(const int8_t* values) : simd8(load(values)) {} + simdjson_really_inline simd8(const int8_t* values) : simd8(load(values)) {} // Member-by-member initialization #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO - really_inline simd8( + simdjson_really_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(make_int8x16_t( @@ -2279,7 +2279,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ v8, v9, v10,v11,v12,v13,v14,v15 )) {} #else - really_inline simd8( + simdjson_really_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(int8x16_t{ @@ -2288,7 +2288,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ }) {} #endif // Repeat 16 values as many times as necessary (usually for lookup tables) - really_inline static simd8 repeat_16( + simdjson_really_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -2299,7 +2299,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ } // Store to array - really_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } + simdjson_really_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } // Explicit conversion to/from unsigned // @@ -2307,35 +2307,35 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ // In theory, we could check this occurence with std::same_as and std::enabled_if but it is C++14 // and relatively ugly and hard to read. #ifndef SIMDJSON_REGULAR_VISUAL_STUDIO - really_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} + simdjson_really_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} #endif - really_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } + simdjson_really_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } // Math - really_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } - really_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } - really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + simdjson_really_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } + simdjson_really_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } + simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } // Order-sensitive comparisons - really_inline simd8 max(const simd8 other) const { return vmaxq_s8(*this, other); } - really_inline simd8 min(const simd8 other) const { return vminq_s8(*this, other); } - really_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } - really_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } - really_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } + simdjson_really_inline simd8 max(const simd8 other) const { return vmaxq_s8(*this, other); } + simdjson_really_inline simd8 min(const simd8 other) const { return vminq_s8(*this, other); } + simdjson_really_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } + simdjson_really_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } template - really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { return vextq_s8(prev_chunk, *this, 16 - N); } // Perform a lookup assuming no value is larger than 16 template - really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { return lookup_table.apply_lookup_16_to(*this); } template - really_inline simd8 lookup_16( + simdjson_really_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -2349,7 +2349,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ } template - really_inline simd8 apply_lookup_16_to(const simd8 original) { + simdjson_really_inline simd8 apply_lookup_16_to(const simd8 original) { return vqtbl1q_s8(*this, simd8(original)); } }; @@ -2364,29 +2364,29 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ simd8x64& operator=(const simd8 other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - really_inline void store(T ptr[64]) const { + simdjson_really_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); this->chunks[2].store(ptr+sizeof(simd8)*2); this->chunks[3].store(ptr+sizeof(simd8)*3); } - really_inline simd8 reduce_or() const { + simdjson_really_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - really_inline void compress(uint64_t mask, T * output) const { + simdjson_really_inline void compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); } - really_inline uint64_t to_bitmask() const { + simdjson_really_inline uint64_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const uint8x16_t bit_mask = make_uint8x16_t( 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, @@ -2406,7 +2406,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); } - really_inline simd8x64 bit_or(const T m) const { + simdjson_really_inline simd8x64 bit_or(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] | mask, @@ -2416,7 +2416,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ ); } - really_inline uint64_t eq(const T m) const { + simdjson_really_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -2426,7 +2426,7 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ ).to_bitmask(); } - really_inline uint64_t lteq(const T m) const { + simdjson_really_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -2475,17 +2475,17 @@ public: /** Document passed to stage 2 */ dom::document *doc{}; - really_inline dom_parser_implementation(); + simdjson_really_inline dom_parser_implementation(); dom_parser_implementation(const dom_parser_implementation &) = delete; dom_parser_implementation & operator=(const dom_parser_implementation &) = delete; - WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; - WARN_UNUSED error_code check_for_unclosed_array() noexcept; - WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; - WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; - WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; - WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; + SIMDJSON_WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; + SIMDJSON_WARN_UNUSED error_code check_for_unclosed_array() noexcept; + SIMDJSON_WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; }; } // namespace SIMDJSON_IMPLEMENTATION @@ -2500,8 +2500,8 @@ namespace allocate { // // Allocates stage 1 internal state and outputs in the parser // -really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { - size_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7; +simdjson_really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; parser.structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!parser.structural_indexes) { return MEMALLOC; } parser.structural_indexes[0] = 0; @@ -2523,7 +2523,7 @@ namespace allocate { // // Allocates stage 2 internal state and outputs in the parser // -really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { +simdjson_really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { parser.open_containers.reset(new (std::nothrow) open_container[max_depth]); parser.is_array.reset(new (std::nothrow) bool[max_depth]); @@ -2542,17 +2542,17 @@ really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t namespace { namespace SIMDJSON_IMPLEMENTATION { -really_inline dom_parser_implementation::dom_parser_implementation() {} +simdjson_really_inline dom_parser_implementation::dom_parser_implementation() {} // Leaving these here so they can be inlined if so desired -WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { error_code err = stage1::allocate::set_capacity(*this, capacity); if (err) { _capacity = 0; return err; } _capacity = capacity; return SUCCESS; } -WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { error_code err = stage2::allocate::set_max_depth(*this, max_depth); if (err) { _max_depth = 0; return err; } _max_depth = max_depth; @@ -2569,7 +2569,7 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth namespace { namespace SIMDJSON_IMPLEMENTATION { -WARN_UNUSED error_code implementation::create_dom_parser_implementation( +SIMDJSON_WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, size_t max_depth, std::unique_ptr& dst @@ -2605,11 +2605,11 @@ namespace stage2 { // return non-zero if not a structural or whitespace char // zero otherwise -really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return structural_or_whitespace_negated[c]; } -really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { return structural_or_whitespace[c]; } @@ -2640,7 +2640,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -2672,10 +2672,10 @@ really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -2687,7 +2687,7 @@ static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { } #endif -really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -2719,17 +2719,17 @@ namespace SIMDJSON_IMPLEMENTATION { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); - really_inline uint64_t whitespace() const { return _whitespace; } - really_inline uint64_t op() const { return _op; } - really_inline uint64_t scalar() { return ~(op() | whitespace()); } + simdjson_really_inline uint64_t whitespace() const { return _whitespace; } + simdjson_really_inline uint64_t op() const { return _op; } + simdjson_really_inline uint64_t scalar() { return ~(op() | whitespace()); } uint64_t _whitespace; uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // Functional programming causes trouble with Visual Studio. // Keeping this version in comments since it is much nicer: // auto v = in.map([&](simd8 chunk) { @@ -2783,12 +2783,12 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(const simd8x64& input) { +simdjson_really_inline bool is_ascii(const simd8x64& input) { simd8 bits = input.reduce_or(); return bits.max() < 0b10000000u; } -UNUSED really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { +SIMDJSON_UNUSED simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1 >= uint8_t(0b11000000u); simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); @@ -2800,7 +2800,7 @@ UNUSED really_inline simd8 must_be_continuation(const simd8 prev1 return is_second_byte ^ is_third_byte ^ is_fourth_byte; } -really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { +simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); return is_third_byte ^ is_fourth_byte; @@ -2816,7 +2816,7 @@ namespace utf8_validation { using namespace simd; - really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -2906,7 +2906,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -2919,7 +2919,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(const simd8 input) { + simdjson_really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -2943,7 +2943,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -2952,14 +2952,14 @@ using namespace simd; } // The only problem that can happen at EOF is that a multibyte character is too short. - really_inline void check_eof() { + simdjson_really_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - really_inline void check_next_input(const simd8x64& input) { - if(likely(is_ascii(input))) { + simdjson_really_inline void check_next_input(const simd8x64& input) { + if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. @@ -2980,7 +2980,7 @@ using namespace simd; } } - really_inline error_code errors() { + simdjson_really_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -3006,10 +3006,10 @@ namespace SIMDJSON_IMPLEMENTATION { template struct buf_block_reader { public: - really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - really_inline size_t block_index(); - really_inline bool has_full_block() const; - really_inline const uint8_t *full_block() const; + simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_really_inline size_t block_index(); + simdjson_really_inline bool has_full_block() const; + simdjson_really_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -3019,8 +3019,8 @@ public: * * @return the number of effective characters in the last block. */ - really_inline size_t get_remainder(uint8_t *dst) const; - really_inline void advance(); + simdjson_really_inline size_t get_remainder(uint8_t *dst) const; + simdjson_really_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -3029,7 +3029,7 @@ private: }; // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text_64(const uint8_t *text) { +SIMDJSON_UNUSED static char * format_input_text_64(const uint8_t *text) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); for (size_t i=0; i); i++) { buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); @@ -3039,7 +3039,7 @@ UNUSED static char * format_input_text_64(const uint8_t *text) { } // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text(const simd8x64& in) { +SIMDJSON_UNUSED static char * format_input_text(const simd8x64& in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); in.store((uint8_t*)buf); for (size_t i=0; i); i++) { @@ -3049,7 +3049,7 @@ UNUSED static char * format_input_text(const simd8x64& in) { return buf; } -UNUSED static char * format_mask(uint64_t mask) { +SIMDJSON_UNUSED static char * format_mask(uint64_t mask) { static char *buf = (char*)malloc(64 + 1); for (size_t i=0; i<64; i++) { buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; @@ -3059,30 +3059,30 @@ UNUSED static char * format_mask(uint64_t mask) { } template -really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } template -really_inline bool buf_block_reader::has_full_block() const { +simdjson_really_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { memset(dst, 0x20, STEP_SIZE); // memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. memcpy(dst, buf + idx, len - idx); return len - idx; } template -really_inline void buf_block_reader::advance() { +simdjson_really_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -3096,23 +3096,23 @@ namespace stage1 { struct json_string_block { // Escaped characters (characters following an escape() character) - really_inline uint64_t escaped() const { return _escaped; } + simdjson_really_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - really_inline uint64_t quote() const { return _quote; } + simdjson_really_inline uint64_t quote() const { return _quote; } // Start quotes of strings - really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -3127,13 +3127,13 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - really_inline json_string_block next(const simd::simd8x64& in); - really_inline error_code finish(bool streaming); + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_really_inline error_code finish(bool streaming); private: // Intended to be defined by the implementation - really_inline uint64_t find_escaped(uint64_t escape); - really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_really_inline uint64_t find_escaped(uint64_t escape); + simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -3168,7 +3168,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -3193,7 +3193,7 @@ really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t bac // // Backslash sequences outside of quotes will be detected in stage 2. // -really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -3223,7 +3223,7 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in); - really_inline error_code finish(bool streaming); + simdjson_really_inline json_block next(const simd::simd8x64& in); + simdjson_really_inline error_code finish(bool streaming); private: // Whether the last character of the previous iteration is part of a scalar token @@ -3345,13 +3345,13 @@ private: // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structurat characters json_character_block characters = json_character_block::classify(in); @@ -3374,7 +3374,7 @@ really_inline json_block json_scanner::next(const simd::simd8x64& in) { }; } -really_inline error_code json_scanner::finish(bool streaming) { +simdjson_really_inline error_code json_scanner::finish(bool streaming) { return string_scanner.finish(streaming); } @@ -3398,24 +3398,24 @@ public: static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; private: - really_inline json_minifier(uint8_t *_dst) + simdjson_really_inline json_minifier(uint8_t *_dst) : dst{_dst} {} template - really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - really_inline void next(const simd::simd8x64& in, json_block block); - really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_really_inline void next(const simd::simd8x64& in, json_block block); + simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { +simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { uint64_t mask = block.whitespace(); in.compress(mask, dst); dst += 64 - count_ones(mask); } -really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { *dst = '\0'; error_code error = scanner.finish(false); if (error) { dst_len = 0; return error; } @@ -3424,7 +3424,7 @@ really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_l } template<> -really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -3435,7 +3435,7 @@ really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_ } template<> -really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -3454,7 +3454,7 @@ error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, s // Index the last (remainder) block, padded with spaces uint8_t block[STEP_SIZE]; - if (likely(reader.get_remainder(block)) > 0) { + if (simdjson_likely(reader.get_remainder(block)) > 0) { minifier.step(block, reader); } @@ -3494,7 +3494,7 @@ namespace SIMDJSON_IMPLEMENTATION { * complete document, therefore the last json buffer location is the end of the * batch. */ -really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // TODO don't count separately, just figure out depth auto arr_cnt = 0; auto obj_cnt = 0; @@ -3547,14 +3547,14 @@ class bit_indexer { public: uint32_t *tail; - really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this - really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -3570,7 +3570,7 @@ public: // Do the next 8 all together (we hope in most cases it won't happen at all // and the branch is easily predicted). - if (unlikely(cnt > 8)) { + if (simdjson_unlikely(cnt > 8)) { for (int i=8; i<16; i++) { this->tail[i] = idx + trailing_zeroes(bits); bits = clear_lowest_bit(bits); @@ -3579,7 +3579,7 @@ public: // Most files don't have 16+ structurals per block, so we take several basically guaranteed // branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :) // or the start of a value ("abc" true 123) every four characters. - if (unlikely(cnt > 16)) { + if (simdjson_unlikely(cnt > 16)) { int i = 16; do { this->tail[i] = idx + trailing_zeroes(bits); @@ -3606,11 +3606,11 @@ public: static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, bool partial) noexcept; private: - really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); template - really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); - really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); + simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); + simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); json_scanner scanner{}; utf8_checker checker{}; @@ -3619,11 +3619,11 @@ private: uint64_t unescaped_chars_error = 0; }; -really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (unlikely(len < 3)) { +simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { + if (simdjson_unlikely(len < 3)) { switch (len) { case 2: if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left @@ -3661,7 +3661,7 @@ really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { // template error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, bool partial) noexcept { - if (unlikely(len > parser.capacity())) { return CAPACITY; } + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } if (partial) { len = trim_partial_utf8(buf, len); } buf_block_reader reader(buf, len); @@ -3674,14 +3674,14 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa // Take care of the last block (will always be there unless file is empty) uint8_t block[STEP_SIZE]; - if (unlikely(reader.get_remainder(block) == 0)) { return EMPTY; } + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return EMPTY; } indexer.step(block, reader); return indexer.finish(parser, reader.block_index(), len, partial); } template<> -really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -3692,14 +3692,14 @@ really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_ } template<> -really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { +simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -3707,12 +3707,12 @@ really_inline void json_structural_indexer::next(const simd::simd8x64& unescaped_chars_error |= block.non_quote_inside_string(unescaped); } -really_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial) { +simdjson_really_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial) { // Write out the final iteration's structurals indexer.write(uint32_t(idx-64), prev_structurals); error_code error = scanner.finish(partial); - if (unlikely(error != SUCCESS)) { return error; } + if (simdjson_unlikely(error != SUCCESS)) { return error; } if (unescaped_chars_error) { return UNESCAPED_CHARS; @@ -3738,10 +3738,10 @@ really_inline error_code json_structural_indexer::finish(dom_parser_implementati parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; // a valid JSON file cannot have zero structural indexes - we should have found something - if (unlikely(parser.n_structural_indexes == 0u)) { + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - if (unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { return UNEXPECTED_ERROR; } if (partial) { @@ -3812,18 +3812,18 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - really_inline bool has_backslash() { return bs_bits != 0; } - really_inline int quote_index() { return trailing_zeroes(quote_bits); } - really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_really_inline bool has_backslash() { return bs_bits != 0; } + simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } uint32_t bs_bits; uint32_t quote_bits; }; // struct backslash_and_quote -really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); @@ -3885,8 +3885,8 @@ static const uint8_t escape_map[256] = { // dest will advance a variable amount (return via pointer) // return true if the unicode codepoint was valid // We work in little-endian then swap at write time -WARN_UNUSED -really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **dst_ptr) { // hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the @@ -3919,7 +3919,7 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, return offset > 0; } -WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +SIMDJSON_WARN_UNUSED simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { src++; while (1) { // Copy the next n bytes, and find the backslash and quote in them. @@ -3966,7 +3966,7 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst return nullptr; } -UNUSED WARN_UNUSED really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { +SIMDJSON_UNUSED SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { if (src[0] != '"') { return STRING_ERROR; } auto end = stringparsing::parse_string(src, current_string_buf_loc); if (!end) { return STRING_ERROR; } @@ -3992,7 +3992,7 @@ namespace arm64 { // we don't have SSE, so let us use a scalar function // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { uint64_t val; memcpy(&val, chars, sizeof(uint64_t)); val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; @@ -4032,7 +4032,7 @@ namespace numberparsing { // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [FASTFLOAT_SMALLEST_POWER, // FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. -really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -4125,7 +4125,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // know that we have an exact computed value for the leading // 55 bits because any imprecision would play out as a +1, in // the worst case. - if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + if (simdjson_unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { uint64_t factor_mantissa_low = mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit @@ -4162,7 +4162,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // which we guard against. // If we have lots of trailing zeros, we may fall right between two // floating-point values. - if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + if (simdjson_unlikely((lower == 0) && ((upper & 0x1FF) == 0) && ((mantissa & 3) == 1))) { // if mantissa & 1 == 1 we might need to round up. // @@ -4198,7 +4198,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do mantissa &= ~(1ULL << 52); uint64_t real_exponent = c.exp - lz; // we have to check that real_exponent is in range, otherwise we bail out - if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + if (simdjson_unlikely((real_exponent < 1) || (real_exponent > 2046))) { return false; } mantissa |= real_exponent << 52; @@ -4239,7 +4239,7 @@ static bool parse_float_strtod(const uint8_t *ptr, double *outDouble) { // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -4255,7 +4255,7 @@ really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { } template -error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { +error_code slow_float_parsing(SIMDJSON_UNUSED const uint8_t * src, W writer) { double d; if (parse_float_strtod(src, &d)) { writer.append_double(d); @@ -4266,7 +4266,7 @@ error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { template NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -4276,7 +4276,7 @@ really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_really_inline error_code parse_decimal(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -4302,7 +4302,7 @@ really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const ui return SUCCESS; } -really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_really_inline error_code parse_exponent(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -4315,14 +4315,14 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. // Thus we *must* check for possible overflow before we negate exp_number. - // Performance notes: it may seem like combining the two "unlikely checks" below into - // a single unlikely path would be faster. The reasoning is sound, but the compiler may + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may // not oblige and may, in fact, generate two distinct paths in any case. It might be // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a likely branch, an unconclusive gain. + // instructions for a simdjson_likely branch, an unconclusive gain. // If there were no digits, it's an error. - if (unlikely(p == start_exp)) { + if (simdjson_unlikely(p == start_exp)) { return INVALID_NUMBER(src); } // We have a valid positive exponent in exp_number at this point, except that @@ -4330,7 +4330,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! - if (unlikely(p > start_exp+18)) { + if (simdjson_unlikely(p > start_exp+18)) { // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't @@ -4353,7 +4353,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u return SUCCESS; } -really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { +simdjson_really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -4365,12 +4365,12 @@ really_inline int significant_digits(const uint8_t * start_digits, int digit_cou } template -really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { +simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. // digit count is off by 1 because of the decimal (assuming there was one). - if (unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { + if (simdjson_unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -4387,10 +4387,10 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui writer.skip_double(); return error; } - // NOTE: it's weird that the unlikely() only wraps half the if, but it seems to get slower any other + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { + if (simdjson_unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { // this is almost never going to get called!!! // we start anew, going slowly!!! // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens @@ -4415,7 +4415,7 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui #ifdef SIMDJSON_SKIPNUMBERPARSING template -really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } @@ -4432,7 +4432,7 @@ really_inline error_code parse_number(const uint8_t *const, W &writer) { // // Our objective is accurate parsing (ULP of 0) at high speed. template -really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign @@ -4515,7 +4515,7 @@ really_inline error_code parse_number(const uint8_t *const src, W &writer) { // SAX functions namespace { // Parse any number from 0 to 18,446,744,073,709,551,615 -UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -4555,7 +4555,7 @@ UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * co // Parse any number from 0 to 18,446,744,073,709,551,615 // Call this version of the method if you regularly expect 8- or 16-digit numbers. -UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -4609,7 +4609,7 @@ UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // @@ -4658,7 +4658,7 @@ UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) return negative ? (~i+1) : i; } -UNUSED really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // @@ -4681,7 +4681,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // int64_t exponent = 0; bool overflow; - if (likely(*p == '.')) { + if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits @@ -4691,7 +4691,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Overflow check. 19 digits (minus the decimal) may be overflow. overflow = p-src-1 >= 19; - if (unlikely(overflow && leading_zero)) { + if (simdjson_unlikely(overflow && leading_zero)) { // Skip leading 0.00000 and see if it still overflows const uint8_t *start_digits = src + 2; while (*start_digits == '0') { start_digits++; } @@ -4723,7 +4723,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Assemble (or slow-parse) the float // double d; - if (likely(!overflow)) { + if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } if (!parse_float_strtod(src-negative, &d)) { @@ -4762,7 +4762,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static really_inline char printable_char(char c) { + static simdjson_really_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -4771,7 +4771,7 @@ namespace logger { } // Print the header and set up log_start - static really_inline void log_start() { + static simdjson_really_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -4782,7 +4782,7 @@ namespace logger { // Logs a single line of template - static really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -4858,14 +4858,14 @@ public: * - increment_count(iter) - each time a value is found in an array or object. */ template - WARN_UNUSED really_inline error_code walk_document(V &visitor) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -4874,7 +4874,7 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *peek() const noexcept; + simdjson_really_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -4882,56 +4882,56 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *advance() noexcept; + simdjson_really_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - really_inline size_t remaining_len() const noexcept; + simdjson_really_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - really_inline bool at_eof() const noexcept; + simdjson_really_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - really_inline bool at_beginning() const noexcept; - really_inline uint8_t last_structural() const noexcept; + simdjson_really_inline bool at_beginning() const noexcept; + simdjson_really_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_value(const char *type) const noexcept; + simdjson_really_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_start_value(const char *type) const noexcept; + simdjson_really_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_end_value(const char *type) const noexcept; + simdjson_really_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_error(const char *error) const noexcept; + simdjson_really_inline void log_error(const char *error) const noexcept; template - WARN_UNUSED really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - WARN_UNUSED really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -WARN_UNUSED really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -4982,7 +4982,7 @@ object_begin: } object_field: - if (unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { auto value = advance(); switch (*value) { @@ -4998,7 +4998,7 @@ object_continue: SIMDJSON_TRY( visitor.increment_count(*this) ); { auto key = advance(); - if (unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } goto object_field; @@ -5056,52 +5056,52 @@ document_end: } // walk_document() -really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -really_inline bool json_iterator::at_eof() const noexcept { +simdjson_really_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_really_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -5117,7 +5117,7 @@ WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visi } } template -WARN_UNUSED really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -5147,18 +5147,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - really_inline void append_s64(int64_t value) noexcept; + simdjson_really_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - really_inline void append_u64(uint64_t value) noexcept; + simdjson_really_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - really_inline void append_double(double value) noexcept; + simdjson_really_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -5166,24 +5166,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - really_inline void skip() noexcept; + simdjson_really_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - really_inline void skip_large_integer() noexcept; + simdjson_really_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - really_inline void skip_double() noexcept; + simdjson_really_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -5191,50 +5191,50 @@ private: * all 64 bits, such as double and uint64_t. */ template - really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -really_inline void tape_writer::append_double(double value) noexcept { +simdjson_really_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -really_inline void tape_writer::skip() noexcept { +simdjson_really_inline void tape_writer::skip() noexcept { next_tape_loc++; } -really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_really_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::skip_double() noexcept { +simdjson_really_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -5254,50 +5254,50 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -WARN_UNUSED -really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); return srcval ^ string_to_uint32(atom); } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -5315,40 +5315,40 @@ namespace stage2 { struct tape_builder { template - WARN_UNUSED static really_inline error_code parse_document( + SIMDJSON_WARN_UNUSED static simdjson_really_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - WARN_UNUSED really_inline error_code visit_document_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - WARN_UNUSED really_inline error_code visit_document_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - WARN_UNUSED really_inline error_code visit_array_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - WARN_UNUSED really_inline error_code visit_array_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - WARN_UNUSED really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - WARN_UNUSED really_inline error_code visit_object_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - WARN_UNUSED really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - WARN_UNUSED really_inline error_code visit_object_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - WARN_UNUSED really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - WARN_UNUSED really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -5357,22 +5357,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - WARN_UNUSED really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - WARN_UNUSED really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - WARN_UNUSED really_inline error_code increment_count(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -5380,18 +5380,18 @@ private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - really_inline tape_builder(dom::document &doc) noexcept; + simdjson_really_inline tape_builder(dom::document &doc) noexcept; - really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - really_inline void start_container(json_iterator &iter) noexcept; - WARN_UNUSED really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - WARN_UNUSED really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_really_inline void start_container(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -WARN_UNUSED really_inline error_code tape_builder::parse_document( +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -5400,56 +5400,56 @@ WARN_UNUSED really_inline error_code tape_builder::parse_document( return iter.walk_document(builder); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -WARN_UNUSED really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value, dst); @@ -5461,16 +5461,16 @@ WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &i return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -5493,42 +5493,42 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterat return error; } -WARN_UNUSED really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -5537,24 +5537,24 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_ite // private: -really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -WARN_UNUSED really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -5567,13 +5567,13 @@ WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator & return SUCCESS; } -really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -5597,7 +5597,7 @@ namespace { namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no // benefit and therefore makes things worse. // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } @@ -5606,29 +5606,29 @@ really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { } // namespace stage1 -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { +SIMDJSON_WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); } -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { this->buf = _buf; this->len = _len; return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); } -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { +SIMDJSON_WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return arm64::stage1::generic_validate_utf8(buf,len); } -WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, false); if (error) { return error; } return stage2(_doc); @@ -5674,7 +5674,7 @@ static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { #endif /* result might be undefined when input_num is zero */ -really_inline int leading_zeroes(uint64_t input_num) { +simdjson_really_inline int leading_zeroes(uint64_t input_num) { #ifdef _MSC_VER unsigned long leading_zero = 0; // Search the mask data from most significant bit (MSB) @@ -5725,17 +5725,17 @@ public: /** Document passed to stage 2 */ dom::document *doc{}; - really_inline dom_parser_implementation(); + simdjson_really_inline dom_parser_implementation(); dom_parser_implementation(const dom_parser_implementation &) = delete; dom_parser_implementation & operator=(const dom_parser_implementation &) = delete; - WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; - WARN_UNUSED error_code check_for_unclosed_array() noexcept; - WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; - WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; - WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; - WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; + SIMDJSON_WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; + SIMDJSON_WARN_UNUSED error_code check_for_unclosed_array() noexcept; + SIMDJSON_WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; }; } // namespace SIMDJSON_IMPLEMENTATION @@ -5750,8 +5750,8 @@ namespace allocate { // // Allocates stage 1 internal state and outputs in the parser // -really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { - size_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7; +simdjson_really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; parser.structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!parser.structural_indexes) { return MEMALLOC; } parser.structural_indexes[0] = 0; @@ -5773,7 +5773,7 @@ namespace allocate { // // Allocates stage 2 internal state and outputs in the parser // -really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { +simdjson_really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { parser.open_containers.reset(new (std::nothrow) open_container[max_depth]); parser.is_array.reset(new (std::nothrow) bool[max_depth]); @@ -5792,17 +5792,17 @@ really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t namespace { namespace SIMDJSON_IMPLEMENTATION { -really_inline dom_parser_implementation::dom_parser_implementation() {} +simdjson_really_inline dom_parser_implementation::dom_parser_implementation() {} // Leaving these here so they can be inlined if so desired -WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { error_code err = stage1::allocate::set_capacity(*this, capacity); if (err) { _capacity = 0; return err; } _capacity = capacity; return SUCCESS; } -WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { error_code err = stage2::allocate::set_max_depth(*this, max_depth); if (err) { _max_depth = 0; return err; } _max_depth = max_depth; @@ -5819,7 +5819,7 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth namespace { namespace SIMDJSON_IMPLEMENTATION { -WARN_UNUSED error_code implementation::create_dom_parser_implementation( +SIMDJSON_WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, size_t max_depth, std::unique_ptr& dst @@ -5852,11 +5852,11 @@ namespace stage2 { // return non-zero if not a structural or whitespace char // zero otherwise -really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return structural_or_whitespace_negated[c]; } -really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { return structural_or_whitespace[c]; } @@ -5887,7 +5887,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -5919,10 +5919,10 @@ really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -5934,7 +5934,7 @@ static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { } #endif -really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -5989,7 +5989,7 @@ namespace SIMDJSON_IMPLEMENTATION { * complete document, therefore the last json buffer location is the end of the * batch. */ -really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // TODO don't count separately, just figure out depth auto arr_cnt = 0; auto obj_cnt = 0; @@ -6041,7 +6041,7 @@ namespace stage1 { class structural_scanner { public: -really_inline structural_scanner(dom_parser_implementation &_parser, bool _partial) +simdjson_really_inline structural_scanner(dom_parser_implementation &_parser, bool _partial) : buf{_parser.buf}, next_structural_index{_parser.structural_indexes.get()}, parser{_parser}, @@ -6049,18 +6049,18 @@ really_inline structural_scanner(dom_parser_implementation &_parser, bool _parti partial{_partial} { } -really_inline void add_structural() { +simdjson_really_inline void add_structural() { *next_structural_index = idx; next_structural_index++; } -really_inline bool is_continuation(uint8_t c) { +simdjson_really_inline bool is_continuation(uint8_t c) { return (c & 0b11000000) == 0b10000000; } -really_inline void validate_utf8_character() { +simdjson_really_inline void validate_utf8_character() { // Continuation - if (unlikely((buf[idx] & 0b01000000) == 0)) { + if (simdjson_unlikely((buf[idx] & 0b01000000) == 0)) { // extra continuation error = UTF8_ERROR; idx++; @@ -6070,7 +6070,7 @@ really_inline void validate_utf8_character() { // 2-byte if ((buf[idx] & 0b00100000) == 0) { // missing continuation - if (unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) { + if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) { if (idx+1 > len && partial) { idx = len; return; } error = UTF8_ERROR; idx++; @@ -6085,7 +6085,7 @@ really_inline void validate_utf8_character() { // 3-byte if ((buf[idx] & 0b00010000) == 0) { // missing continuation - if (unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { + if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { if (idx+2 > len && partial) { idx = len; return; } error = UTF8_ERROR; idx++; @@ -6101,7 +6101,7 @@ really_inline void validate_utf8_character() { // 4-byte // missing continuation - if (unlikely(idx+3 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { + if (simdjson_unlikely(idx+3 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { if (idx+2 > len && partial) { idx = len; return; } error = UTF8_ERROR; idx++; @@ -6119,12 +6119,12 @@ really_inline void validate_utf8_character() { idx += 4; } -really_inline void validate_string() { +simdjson_really_inline void validate_string() { idx++; // skip first quote while (idx < len && buf[idx] != '"') { if (buf[idx] == '\\') { idx += 2; - } else if (unlikely(buf[idx] & 0b10000000)) { + } else if (simdjson_unlikely(buf[idx] & 0b10000000)) { validate_utf8_character(); } else { if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; } @@ -6134,7 +6134,7 @@ really_inline void validate_string() { if (idx >= len && !partial) { error = UNCLOSED_STRING; } } -really_inline bool is_whitespace_or_operator(uint8_t c) { +simdjson_really_inline bool is_whitespace_or_operator(uint8_t c) { switch (c) { case '{': case '}': case '[': case ']': case ',': case ':': case ' ': case '\r': case '\n': case '\t': @@ -6147,7 +6147,7 @@ really_inline bool is_whitespace_or_operator(uint8_t c) { // // Parse the entire input in STEP_SIZE-byte chunks. // -really_inline error_code scan() { +simdjson_really_inline error_code scan() { for (;idxbuf = _buf; this->len = _len; stage1::structural_scanner scanner(*this, partial); @@ -6249,7 +6249,7 @@ static uint8_t jump_table[256 * 3] = { 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, }; -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { +SIMDJSON_WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { size_t i = 0, pos = 0; uint8_t quote = 0; uint8_t nonescape = 1; @@ -6271,7 +6271,7 @@ WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, ui } // credit: based on code from Google Fuchsia (Apache Licensed) -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { +SIMDJSON_WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { const uint8_t *data = (const uint8_t *)buf; uint64_t pos = 0; uint64_t next_pos = 0; @@ -6352,17 +6352,17 @@ namespace fallback { struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 1; - really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - really_inline bool has_quote_first() { return c == '"'; } - really_inline bool has_backslash() { return c == '\\'; } - really_inline int quote_index() { return c == '"' ? 0 : 1; } - really_inline int backslash_index() { return c == '\\' ? 0 : 1; } + simdjson_really_inline bool has_quote_first() { return c == '"'; } + simdjson_really_inline bool has_backslash() { return c == '\\'; } + simdjson_really_inline int quote_index() { return c == '"' ? 0 : 1; } + simdjson_really_inline int backslash_index() { return c == '\\' ? 0 : 1; } uint8_t c; }; // struct backslash_and_quote -really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // store to dest unconditionally - we can overwrite the bits we don't like later dst[0] = src[0]; return { src[0] }; @@ -6412,8 +6412,8 @@ static const uint8_t escape_map[256] = { // dest will advance a variable amount (return via pointer) // return true if the unicode codepoint was valid // We work in little-endian then swap at write time -WARN_UNUSED -really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **dst_ptr) { // hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the @@ -6446,7 +6446,7 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, return offset > 0; } -WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +SIMDJSON_WARN_UNUSED simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { src++; while (1) { // Copy the next n bytes, and find the backslash and quote in them. @@ -6493,7 +6493,7 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst return nullptr; } -UNUSED WARN_UNUSED really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { +SIMDJSON_UNUSED SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { if (src[0] != '"') { return STRING_ERROR; } auto end = stringparsing::parse_string(src, current_string_buf_loc); if (!end) { return STRING_ERROR; } @@ -6523,14 +6523,14 @@ void found_float(double result, const uint8_t *buf); namespace { namespace SIMDJSON_IMPLEMENTATION { -static really_inline uint32_t parse_eight_digits_unrolled(const char *chars) { +static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const char *chars) { uint32_t result = 0; for (int i=0;i<8;i++) { result = result*10 + (chars[i] - '0'); } return result; } -static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { return parse_eight_digits_unrolled((const char *)chars); } @@ -6565,7 +6565,7 @@ namespace numberparsing { // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [FASTFLOAT_SMALLEST_POWER, // FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. -really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -6658,7 +6658,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // know that we have an exact computed value for the leading // 55 bits because any imprecision would play out as a +1, in // the worst case. - if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + if (simdjson_unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { uint64_t factor_mantissa_low = mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit @@ -6695,7 +6695,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // which we guard against. // If we have lots of trailing zeros, we may fall right between two // floating-point values. - if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + if (simdjson_unlikely((lower == 0) && ((upper & 0x1FF) == 0) && ((mantissa & 3) == 1))) { // if mantissa & 1 == 1 we might need to round up. // @@ -6731,7 +6731,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do mantissa &= ~(1ULL << 52); uint64_t real_exponent = c.exp - lz; // we have to check that real_exponent is in range, otherwise we bail out - if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + if (simdjson_unlikely((real_exponent < 1) || (real_exponent > 2046))) { return false; } mantissa |= real_exponent << 52; @@ -6772,7 +6772,7 @@ static bool parse_float_strtod(const uint8_t *ptr, double *outDouble) { // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -6788,7 +6788,7 @@ really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { } template -error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { +error_code slow_float_parsing(SIMDJSON_UNUSED const uint8_t * src, W writer) { double d; if (parse_float_strtod(src, &d)) { writer.append_double(d); @@ -6799,7 +6799,7 @@ error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { template NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -6809,7 +6809,7 @@ really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_really_inline error_code parse_decimal(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -6835,7 +6835,7 @@ really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const ui return SUCCESS; } -really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_really_inline error_code parse_exponent(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -6848,14 +6848,14 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. // Thus we *must* check for possible overflow before we negate exp_number. - // Performance notes: it may seem like combining the two "unlikely checks" below into - // a single unlikely path would be faster. The reasoning is sound, but the compiler may + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may // not oblige and may, in fact, generate two distinct paths in any case. It might be // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a likely branch, an unconclusive gain. + // instructions for a simdjson_likely branch, an unconclusive gain. // If there were no digits, it's an error. - if (unlikely(p == start_exp)) { + if (simdjson_unlikely(p == start_exp)) { return INVALID_NUMBER(src); } // We have a valid positive exponent in exp_number at this point, except that @@ -6863,7 +6863,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! - if (unlikely(p > start_exp+18)) { + if (simdjson_unlikely(p > start_exp+18)) { // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't @@ -6886,7 +6886,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u return SUCCESS; } -really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { +simdjson_really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -6898,12 +6898,12 @@ really_inline int significant_digits(const uint8_t * start_digits, int digit_cou } template -really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { +simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. // digit count is off by 1 because of the decimal (assuming there was one). - if (unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { + if (simdjson_unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -6920,10 +6920,10 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui writer.skip_double(); return error; } - // NOTE: it's weird that the unlikely() only wraps half the if, but it seems to get slower any other + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { + if (simdjson_unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { // this is almost never going to get called!!! // we start anew, going slowly!!! // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens @@ -6948,7 +6948,7 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui #ifdef SIMDJSON_SKIPNUMBERPARSING template -really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } @@ -6965,7 +6965,7 @@ really_inline error_code parse_number(const uint8_t *const, W &writer) { // // Our objective is accurate parsing (ULP of 0) at high speed. template -really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign @@ -7048,7 +7048,7 @@ really_inline error_code parse_number(const uint8_t *const src, W &writer) { // SAX functions namespace { // Parse any number from 0 to 18,446,744,073,709,551,615 -UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -7088,7 +7088,7 @@ UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * co // Parse any number from 0 to 18,446,744,073,709,551,615 // Call this version of the method if you regularly expect 8- or 16-digit numbers. -UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -7142,7 +7142,7 @@ UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // @@ -7191,7 +7191,7 @@ UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) return negative ? (~i+1) : i; } -UNUSED really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // @@ -7214,7 +7214,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // int64_t exponent = 0; bool overflow; - if (likely(*p == '.')) { + if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits @@ -7224,7 +7224,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Overflow check. 19 digits (minus the decimal) may be overflow. overflow = p-src-1 >= 19; - if (unlikely(overflow && leading_zero)) { + if (simdjson_unlikely(overflow && leading_zero)) { // Skip leading 0.00000 and see if it still overflows const uint8_t *start_digits = src + 2; while (*start_digits == '0') { start_digits++; } @@ -7256,7 +7256,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Assemble (or slow-parse) the float // double d; - if (likely(!overflow)) { + if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } if (!parse_float_strtod(src-negative, &d)) { @@ -7295,7 +7295,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static really_inline char printable_char(char c) { + static simdjson_really_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -7304,7 +7304,7 @@ namespace logger { } // Print the header and set up log_start - static really_inline void log_start() { + static simdjson_really_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -7315,7 +7315,7 @@ namespace logger { // Logs a single line of template - static really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -7391,14 +7391,14 @@ public: * - increment_count(iter) - each time a value is found in an array or object. */ template - WARN_UNUSED really_inline error_code walk_document(V &visitor) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -7407,7 +7407,7 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *peek() const noexcept; + simdjson_really_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -7415,56 +7415,56 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *advance() noexcept; + simdjson_really_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - really_inline size_t remaining_len() const noexcept; + simdjson_really_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - really_inline bool at_eof() const noexcept; + simdjson_really_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - really_inline bool at_beginning() const noexcept; - really_inline uint8_t last_structural() const noexcept; + simdjson_really_inline bool at_beginning() const noexcept; + simdjson_really_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_value(const char *type) const noexcept; + simdjson_really_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_start_value(const char *type) const noexcept; + simdjson_really_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_end_value(const char *type) const noexcept; + simdjson_really_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_error(const char *error) const noexcept; + simdjson_really_inline void log_error(const char *error) const noexcept; template - WARN_UNUSED really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - WARN_UNUSED really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -WARN_UNUSED really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -7515,7 +7515,7 @@ object_begin: } object_field: - if (unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { auto value = advance(); switch (*value) { @@ -7531,7 +7531,7 @@ object_continue: SIMDJSON_TRY( visitor.increment_count(*this) ); { auto key = advance(); - if (unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } goto object_field; @@ -7589,52 +7589,52 @@ document_end: } // walk_document() -really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -really_inline bool json_iterator::at_eof() const noexcept { +simdjson_really_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_really_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -7650,7 +7650,7 @@ WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visi } } template -WARN_UNUSED really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -7680,18 +7680,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - really_inline void append_s64(int64_t value) noexcept; + simdjson_really_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - really_inline void append_u64(uint64_t value) noexcept; + simdjson_really_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - really_inline void append_double(double value) noexcept; + simdjson_really_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -7699,24 +7699,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - really_inline void skip() noexcept; + simdjson_really_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - really_inline void skip_large_integer() noexcept; + simdjson_really_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - really_inline void skip_double() noexcept; + simdjson_really_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -7724,50 +7724,50 @@ private: * all 64 bits, such as double and uint64_t. */ template - really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -really_inline void tape_writer::append_double(double value) noexcept { +simdjson_really_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -really_inline void tape_writer::skip() noexcept { +simdjson_really_inline void tape_writer::skip() noexcept { next_tape_loc++; } -really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_really_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::skip_double() noexcept { +simdjson_really_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -7787,50 +7787,50 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -WARN_UNUSED -really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); return srcval ^ string_to_uint32(atom); } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -7848,40 +7848,40 @@ namespace stage2 { struct tape_builder { template - WARN_UNUSED static really_inline error_code parse_document( + SIMDJSON_WARN_UNUSED static simdjson_really_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - WARN_UNUSED really_inline error_code visit_document_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - WARN_UNUSED really_inline error_code visit_document_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - WARN_UNUSED really_inline error_code visit_array_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - WARN_UNUSED really_inline error_code visit_array_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - WARN_UNUSED really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - WARN_UNUSED really_inline error_code visit_object_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - WARN_UNUSED really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - WARN_UNUSED really_inline error_code visit_object_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - WARN_UNUSED really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - WARN_UNUSED really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -7890,22 +7890,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - WARN_UNUSED really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - WARN_UNUSED really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - WARN_UNUSED really_inline error_code increment_count(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -7913,18 +7913,18 @@ private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - really_inline tape_builder(dom::document &doc) noexcept; + simdjson_really_inline tape_builder(dom::document &doc) noexcept; - really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - really_inline void start_container(json_iterator &iter) noexcept; - WARN_UNUSED really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - WARN_UNUSED really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_really_inline void start_container(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -WARN_UNUSED really_inline error_code tape_builder::parse_document( +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -7933,56 +7933,56 @@ WARN_UNUSED really_inline error_code tape_builder::parse_document( return iter.walk_document(builder); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -WARN_UNUSED really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value, dst); @@ -7994,16 +7994,16 @@ WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &i return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -8026,42 +8026,42 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterat return error; } -WARN_UNUSED really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -8070,24 +8070,24 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_ite // private: -really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -WARN_UNUSED really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -8100,13 +8100,13 @@ WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator & return SUCCESS; } -really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -8126,15 +8126,15 @@ really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { namespace { namespace SIMDJSON_IMPLEMENTATION { -WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, false); if (error) { return error; } return stage2(_doc); @@ -8199,7 +8199,7 @@ WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, siz #ifndef _blsr_u64 // we roll our own SIMDJSON_TARGET_HASWELL -static really_inline uint64_t _blsr_u64(uint64_t n) { +static simdjson_really_inline uint64_t _blsr_u64(uint64_t n) { return (n - 1) & n; } SIMDJSON_UNTARGET_REGION @@ -8222,7 +8222,7 @@ namespace SIMDJSON_IMPLEMENTATION { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. NO_SANITIZE_UNDEFINED -really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_really_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return (int)_tzcnt_u64(input_num); #else // SIMDJSON_REGULAR_VISUAL_STUDIO @@ -8236,27 +8236,27 @@ really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { return _blsr_u64(input_num); } /* result might be undefined when input_num is zero */ -really_inline int leading_zeroes(uint64_t input_num) { +simdjson_really_inline int leading_zeroes(uint64_t input_num) { return int(_lzcnt_u64(input_num)); } #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO -really_inline unsigned __int64 count_ones(uint64_t input_num) { +simdjson_really_inline unsigned __int64 count_ones(uint64_t input_num) { // note: we do not support legacy 32-bit Windows return __popcnt64(input_num);// Visual Studio wants two underscores } #else -really_inline long long int count_ones(uint64_t input_num) { +simdjson_really_inline long long int count_ones(uint64_t input_num) { return _popcnt64(input_num); } #endif -really_inline bool add_overflow(uint64_t value1, uint64_t value2, +simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return _addcarry_u64(0, value1, value2, @@ -8284,7 +8284,7 @@ namespace SIMDJSON_IMPLEMENTATION { // // For example, prefix_xor(00100100) == 00011100 // -really_inline uint64_t prefix_xor(const uint64_t bitmask) { +simdjson_really_inline uint64_t prefix_xor(const uint64_t bitmask) { // There should be no such thing with a processor supporting avx2 // but not clmul. __m128i all_ones = _mm_set1_epi8('\xFF'); @@ -8313,23 +8313,23 @@ namespace simd { __m256i value; // Zero constructor - really_inline base() : value{__m256i()} {} + simdjson_really_inline base() : value{__m256i()} {} // Conversion from SIMD register - really_inline base(const __m256i _value) : value(_value) {} + simdjson_really_inline base(const __m256i _value) : value(_value) {} // Conversion to SIMD register - really_inline operator const __m256i&() const { return this->value; } - really_inline operator __m256i&() { return this->value; } + simdjson_really_inline operator const __m256i&() const { return this->value; } + simdjson_really_inline operator __m256i&() { return this->value; } // Bit operations - really_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - really_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - really_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - really_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - really_inline Child& operator|=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast | other; return *this_cast; } - really_inline Child& operator&=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast & other; return *this_cast; } - really_inline Child& operator^=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_really_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } + simdjson_really_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } + simdjson_really_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } + simdjson_really_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } + simdjson_really_inline Child& operator|=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast | other; return *this_cast; } + simdjson_really_inline Child& operator&=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast & other; return *this_cast; } + simdjson_really_inline Child& operator^=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast ^ other; return *this_cast; } }; // Forward-declared so they can be used by splat and friends. @@ -8341,15 +8341,15 @@ namespace simd { typedef uint32_t bitmask_t; typedef uint64_t bitmask2_t; - really_inline base8() : base>() {} - really_inline base8(const __m256i _value) : base>(_value) {} + simdjson_really_inline base8() : base>() {} + simdjson_really_inline base8(const __m256i _value) : base>(_value) {} - really_inline Mask operator==(const simd8 other) const { return _mm256_cmpeq_epi8(*this, other); } + simdjson_really_inline Mask operator==(const simd8 other) const { return _mm256_cmpeq_epi8(*this, other); } static const int SIZE = sizeof(base::value); template - really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); } }; @@ -8357,27 +8357,27 @@ namespace simd { // SIMD byte mask type (returned by things like eq and gt) template<> struct simd8: base8 { - static really_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_really_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - really_inline simd8() : base8() {} - really_inline simd8(const __m256i _value) : base8(_value) {} + simdjson_really_inline simd8() : base8() {} + simdjson_really_inline simd8(const __m256i _value) : base8(_value) {} // Splat constructor - really_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_really_inline simd8(bool _value) : base8(splat(_value)) {} - really_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } - really_inline bool any() const { return !_mm256_testz_si256(*this, *this); } - really_inline simd8 operator~() const { return *this ^ true; } + simdjson_really_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } + simdjson_really_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + simdjson_really_inline simd8 operator~() const { return *this ^ true; } }; template struct base8_numeric: base8 { - static really_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static really_inline simd8 zero() { return _mm256_setzero_si256(); } - static really_inline simd8 load(const T values[32]) { + static simdjson_really_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } + static simdjson_really_inline simd8 zero() { return _mm256_setzero_si256(); } + static simdjson_really_inline simd8 load(const T values[32]) { return _mm256_loadu_si256(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static really_inline simd8 repeat_16( + static simdjson_really_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -8389,24 +8389,24 @@ namespace simd { ); } - really_inline base8_numeric() : base8() {} - really_inline base8_numeric(const __m256i _value) : base8(_value) {} + simdjson_really_inline base8_numeric() : base8() {} + simdjson_really_inline base8_numeric(const __m256i _value) : base8(_value) {} // Store to array - really_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } + simdjson_really_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } // Addition/subtraction are the same for signed and unsigned - really_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - really_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *(simd8*)this; } - really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *(simd8*)this; } + simdjson_really_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } + simdjson_really_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } + simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *(simd8*)this; } + simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *(simd8*)this; } // Override to distinguish from bool version - really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template - really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { return _mm256_shuffle_epi8(lookup_table, *this); } @@ -8418,7 +8418,7 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - really_inline void compress(uint32_t mask, L * output) const { + simdjson_really_inline void compress(uint32_t mask, L * output) const { // this particular implementation was inspired by work done by @animetosho // we do it in four steps, first 8 bytes and then second 8 bytes... uint8_t mask1 = uint8_t(mask); // least significant 8 bits @@ -8461,7 +8461,7 @@ namespace simd { } template - really_inline simd8 lookup_16( + simdjson_really_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -8478,14 +8478,14 @@ namespace simd { // Signed bytes template<> struct simd8 : base8_numeric { - really_inline simd8() : base8_numeric() {} - really_inline simd8(const __m256i _value) : base8_numeric(_value) {} + simdjson_really_inline simd8() : base8_numeric() {} + simdjson_really_inline simd8(const __m256i _value) : base8_numeric(_value) {} // Splat constructor - really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + simdjson_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} // Member-by-member initialization - really_inline simd8( + simdjson_really_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, @@ -8497,7 +8497,7 @@ namespace simd { v24,v25,v26,v27,v28,v29,v30,v31 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - really_inline static simd8 repeat_16( + simdjson_really_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -8510,23 +8510,23 @@ namespace simd { } // Order-sensitive comparisons - really_inline simd8 max(const simd8 other) const { return _mm256_max_epi8(*this, other); } - really_inline simd8 min(const simd8 other) const { return _mm256_min_epi8(*this, other); } - really_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - really_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } + simdjson_really_inline simd8 max(const simd8 other) const { return _mm256_max_epi8(*this, other); } + simdjson_really_inline simd8 min(const simd8 other) const { return _mm256_min_epi8(*this, other); } + simdjson_really_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } }; // Unsigned bytes template<> struct simd8: base8_numeric { - really_inline simd8() : base8_numeric() {} - really_inline simd8(const __m256i _value) : base8_numeric(_value) {} + simdjson_really_inline simd8() : base8_numeric() {} + simdjson_really_inline simd8(const __m256i _value) : base8_numeric(_value) {} // Splat constructor - really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + simdjson_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} // Member-by-member initialization - really_inline simd8( + simdjson_really_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, @@ -8538,7 +8538,7 @@ namespace simd { v24,v25,v26,v27,v28,v29,v30,v31 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - really_inline static simd8 repeat_16( + simdjson_really_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -8551,39 +8551,39 @@ namespace simd { } // Saturated math - really_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - really_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } + simdjson_really_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } + simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } // Order-specific operations - really_inline simd8 max(const simd8 other) const { return _mm256_max_epu8(*this, other); } - really_inline simd8 min(const simd8 other) const { return _mm256_min_epu8(other, *this); } + simdjson_really_inline simd8 max(const simd8 other) const { return _mm256_max_epu8(*this, other); } + simdjson_really_inline simd8 min(const simd8 other) const { return _mm256_min_epu8(other, *this); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - really_inline simd8 operator<=(const simd8 other) const { return other.max(*this) == other; } - really_inline simd8 operator>=(const simd8 other) const { return other.min(*this) == other; } - really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - really_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + simdjson_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_really_inline simd8 operator<=(const simd8 other) const { return other.max(*this) == other; } + simdjson_really_inline simd8 operator>=(const simd8 other) const { return other.min(*this) == other; } + simdjson_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } // Bit-specific operations - really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } + simdjson_really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } + simdjson_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } + simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - really_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_really_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - really_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_really_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - really_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } + simdjson_really_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } }; template @@ -8596,32 +8596,32 @@ namespace simd { simd8x64& operator=(const simd8 other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} + simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} + simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - really_inline void compress(uint64_t mask, T * output) const { + simdjson_really_inline void compress(uint64_t mask, T * output) const { uint32_t mask1 = uint32_t(mask); uint32_t mask2 = uint32_t(mask >> 32); this->chunks[0].compress(mask1, output); this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); } - really_inline void store(T ptr[64]) const { + simdjson_really_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); } - really_inline uint64_t to_bitmask() const { + simdjson_really_inline uint64_t to_bitmask() const { uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); uint64_t r_hi = this->chunks[1].to_bitmask(); return r_lo | (r_hi << 32); } - really_inline simd8 reduce_or() const { + simdjson_really_inline simd8 reduce_or() const { return this->chunks[0] | this->chunks[1]; } - really_inline simd8x64 bit_or(const T m) const { + simdjson_really_inline simd8x64 bit_or(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] | mask, @@ -8629,7 +8629,7 @@ namespace simd { ); } - really_inline uint64_t eq(const T m) const { + simdjson_really_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -8637,7 +8637,7 @@ namespace simd { ).to_bitmask(); } - really_inline uint64_t lteq(const T m) const { + simdjson_really_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -8686,17 +8686,17 @@ public: /** Document passed to stage 2 */ dom::document *doc{}; - really_inline dom_parser_implementation(); + simdjson_really_inline dom_parser_implementation(); dom_parser_implementation(const dom_parser_implementation &) = delete; dom_parser_implementation & operator=(const dom_parser_implementation &) = delete; - WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; - WARN_UNUSED error_code check_for_unclosed_array() noexcept; - WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; - WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; - WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; - WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; + SIMDJSON_WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; + SIMDJSON_WARN_UNUSED error_code check_for_unclosed_array() noexcept; + SIMDJSON_WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; }; } // namespace SIMDJSON_IMPLEMENTATION @@ -8711,8 +8711,8 @@ namespace allocate { // // Allocates stage 1 internal state and outputs in the parser // -really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { - size_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7; +simdjson_really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; parser.structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!parser.structural_indexes) { return MEMALLOC; } parser.structural_indexes[0] = 0; @@ -8734,7 +8734,7 @@ namespace allocate { // // Allocates stage 2 internal state and outputs in the parser // -really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { +simdjson_really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { parser.open_containers.reset(new (std::nothrow) open_container[max_depth]); parser.is_array.reset(new (std::nothrow) bool[max_depth]); @@ -8753,17 +8753,17 @@ really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t namespace { namespace SIMDJSON_IMPLEMENTATION { -really_inline dom_parser_implementation::dom_parser_implementation() {} +simdjson_really_inline dom_parser_implementation::dom_parser_implementation() {} // Leaving these here so they can be inlined if so desired -WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { error_code err = stage1::allocate::set_capacity(*this, capacity); if (err) { _capacity = 0; return err; } _capacity = capacity; return SUCCESS; } -WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { error_code err = stage2::allocate::set_max_depth(*this, max_depth); if (err) { _max_depth = 0; return err; } _max_depth = max_depth; @@ -8780,7 +8780,7 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth namespace { namespace SIMDJSON_IMPLEMENTATION { -WARN_UNUSED error_code implementation::create_dom_parser_implementation( +SIMDJSON_WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, size_t max_depth, std::unique_ptr& dst @@ -8824,11 +8824,11 @@ namespace stage2 { // return non-zero if not a structural or whitespace char // zero otherwise -really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return structural_or_whitespace_negated[c]; } -really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { return structural_or_whitespace[c]; } @@ -8859,7 +8859,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -8891,10 +8891,10 @@ really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -8906,7 +8906,7 @@ static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { } #endif -really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -8939,13 +8939,13 @@ namespace SIMDJSON_IMPLEMENTATION { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); // ASCII white-space ('\r','\n','\t',' ') - really_inline uint64_t whitespace() const { return _whitespace; } + simdjson_really_inline uint64_t whitespace() const { return _whitespace; } // non-quote structural characters (comma, colon, braces, brackets) - really_inline uint64_t op() const { return _op; } + simdjson_really_inline uint64_t op() const { return _op; } // neither a structural character nor a white-space, so letters, numbers and quotes - really_inline uint64_t scalar() { return ~(op() | whitespace()); } + simdjson_really_inline uint64_t scalar() { return ~(op() | whitespace()); } uint64_t _whitespace; // ASCII white-space ('\r','\n','\t',' ') uint64_t _op; // structural characters (comma, colon, braces, brackets but not quotes) @@ -8953,7 +8953,7 @@ struct json_character_block { // This identifies structural characters (comma, colon, braces, brackets), // and ASCII white-space ('\r','\n','\t',' '). -really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -8976,11 +8976,11 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(const simd8x64& input) { +simdjson_really_inline bool is_ascii(const simd8x64& input) { return input.reduce_or().is_ascii(); } -UNUSED really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { +SIMDJSON_UNUSED simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 @@ -8988,7 +8988,7 @@ UNUSED really_inline simd8 must_be_continuation(const simd8 prev1 return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { +simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. @@ -9005,7 +9005,7 @@ namespace utf8_validation { using namespace simd; - really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -9095,7 +9095,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -9108,7 +9108,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(const simd8 input) { + simdjson_really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -9132,7 +9132,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -9141,14 +9141,14 @@ using namespace simd; } // The only problem that can happen at EOF is that a multibyte character is too short. - really_inline void check_eof() { + simdjson_really_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - really_inline void check_next_input(const simd8x64& input) { - if(likely(is_ascii(input))) { + simdjson_really_inline void check_next_input(const simd8x64& input) { + if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. @@ -9169,7 +9169,7 @@ using namespace simd; } } - really_inline error_code errors() { + simdjson_really_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -9195,10 +9195,10 @@ namespace SIMDJSON_IMPLEMENTATION { template struct buf_block_reader { public: - really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - really_inline size_t block_index(); - really_inline bool has_full_block() const; - really_inline const uint8_t *full_block() const; + simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_really_inline size_t block_index(); + simdjson_really_inline bool has_full_block() const; + simdjson_really_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -9208,8 +9208,8 @@ public: * * @return the number of effective characters in the last block. */ - really_inline size_t get_remainder(uint8_t *dst) const; - really_inline void advance(); + simdjson_really_inline size_t get_remainder(uint8_t *dst) const; + simdjson_really_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -9218,7 +9218,7 @@ private: }; // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text_64(const uint8_t *text) { +SIMDJSON_UNUSED static char * format_input_text_64(const uint8_t *text) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); for (size_t i=0; i); i++) { buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); @@ -9228,7 +9228,7 @@ UNUSED static char * format_input_text_64(const uint8_t *text) { } // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text(const simd8x64& in) { +SIMDJSON_UNUSED static char * format_input_text(const simd8x64& in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); in.store((uint8_t*)buf); for (size_t i=0; i); i++) { @@ -9238,7 +9238,7 @@ UNUSED static char * format_input_text(const simd8x64& in) { return buf; } -UNUSED static char * format_mask(uint64_t mask) { +SIMDJSON_UNUSED static char * format_mask(uint64_t mask) { static char *buf = (char*)malloc(64 + 1); for (size_t i=0; i<64; i++) { buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; @@ -9248,30 +9248,30 @@ UNUSED static char * format_mask(uint64_t mask) { } template -really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } template -really_inline bool buf_block_reader::has_full_block() const { +simdjson_really_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { memset(dst, 0x20, STEP_SIZE); // memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. memcpy(dst, buf + idx, len - idx); return len - idx; } template -really_inline void buf_block_reader::advance() { +simdjson_really_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -9285,23 +9285,23 @@ namespace stage1 { struct json_string_block { // Escaped characters (characters following an escape() character) - really_inline uint64_t escaped() const { return _escaped; } + simdjson_really_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - really_inline uint64_t quote() const { return _quote; } + simdjson_really_inline uint64_t quote() const { return _quote; } // Start quotes of strings - really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -9316,13 +9316,13 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - really_inline json_string_block next(const simd::simd8x64& in); - really_inline error_code finish(bool streaming); + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_really_inline error_code finish(bool streaming); private: // Intended to be defined by the implementation - really_inline uint64_t find_escaped(uint64_t escape); - really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_really_inline uint64_t find_escaped(uint64_t escape); + simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -9357,7 +9357,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -9382,7 +9382,7 @@ really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t bac // // Backslash sequences outside of quotes will be detected in stage 2. // -really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -9412,7 +9412,7 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in); - really_inline error_code finish(bool streaming); + simdjson_really_inline json_block next(const simd::simd8x64& in); + simdjson_really_inline error_code finish(bool streaming); private: // Whether the last character of the previous iteration is part of a scalar token @@ -9534,13 +9534,13 @@ private: // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structurat characters json_character_block characters = json_character_block::classify(in); @@ -9563,7 +9563,7 @@ really_inline json_block json_scanner::next(const simd::simd8x64& in) { }; } -really_inline error_code json_scanner::finish(bool streaming) { +simdjson_really_inline error_code json_scanner::finish(bool streaming) { return string_scanner.finish(streaming); } @@ -9587,24 +9587,24 @@ public: static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; private: - really_inline json_minifier(uint8_t *_dst) + simdjson_really_inline json_minifier(uint8_t *_dst) : dst{_dst} {} template - really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - really_inline void next(const simd::simd8x64& in, json_block block); - really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_really_inline void next(const simd::simd8x64& in, json_block block); + simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { +simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { uint64_t mask = block.whitespace(); in.compress(mask, dst); dst += 64 - count_ones(mask); } -really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { *dst = '\0'; error_code error = scanner.finish(false); if (error) { dst_len = 0; return error; } @@ -9613,7 +9613,7 @@ really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_l } template<> -really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -9624,7 +9624,7 @@ really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_ } template<> -really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -9643,7 +9643,7 @@ error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, s // Index the last (remainder) block, padded with spaces uint8_t block[STEP_SIZE]; - if (likely(reader.get_remainder(block)) > 0) { + if (simdjson_likely(reader.get_remainder(block)) > 0) { minifier.step(block, reader); } @@ -9683,7 +9683,7 @@ namespace SIMDJSON_IMPLEMENTATION { * complete document, therefore the last json buffer location is the end of the * batch. */ -really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // TODO don't count separately, just figure out depth auto arr_cnt = 0; auto obj_cnt = 0; @@ -9736,14 +9736,14 @@ class bit_indexer { public: uint32_t *tail; - really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this - really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -9759,7 +9759,7 @@ public: // Do the next 8 all together (we hope in most cases it won't happen at all // and the branch is easily predicted). - if (unlikely(cnt > 8)) { + if (simdjson_unlikely(cnt > 8)) { for (int i=8; i<16; i++) { this->tail[i] = idx + trailing_zeroes(bits); bits = clear_lowest_bit(bits); @@ -9768,7 +9768,7 @@ public: // Most files don't have 16+ structurals per block, so we take several basically guaranteed // branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :) // or the start of a value ("abc" true 123) every four characters. - if (unlikely(cnt > 16)) { + if (simdjson_unlikely(cnt > 16)) { int i = 16; do { this->tail[i] = idx + trailing_zeroes(bits); @@ -9795,11 +9795,11 @@ public: static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, bool partial) noexcept; private: - really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); template - really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); - really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); + simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); + simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); json_scanner scanner{}; utf8_checker checker{}; @@ -9808,11 +9808,11 @@ private: uint64_t unescaped_chars_error = 0; }; -really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (unlikely(len < 3)) { +simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { + if (simdjson_unlikely(len < 3)) { switch (len) { case 2: if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left @@ -9850,7 +9850,7 @@ really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { // template error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, bool partial) noexcept { - if (unlikely(len > parser.capacity())) { return CAPACITY; } + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } if (partial) { len = trim_partial_utf8(buf, len); } buf_block_reader reader(buf, len); @@ -9863,14 +9863,14 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa // Take care of the last block (will always be there unless file is empty) uint8_t block[STEP_SIZE]; - if (unlikely(reader.get_remainder(block) == 0)) { return EMPTY; } + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return EMPTY; } indexer.step(block, reader); return indexer.finish(parser, reader.block_index(), len, partial); } template<> -really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -9881,14 +9881,14 @@ really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_ } template<> -really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { +simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -9896,12 +9896,12 @@ really_inline void json_structural_indexer::next(const simd::simd8x64& unescaped_chars_error |= block.non_quote_inside_string(unescaped); } -really_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial) { +simdjson_really_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial) { // Write out the final iteration's structurals indexer.write(uint32_t(idx-64), prev_structurals); error_code error = scanner.finish(partial); - if (unlikely(error != SUCCESS)) { return error; } + if (simdjson_unlikely(error != SUCCESS)) { return error; } if (unescaped_chars_error) { return UNESCAPED_CHARS; @@ -9927,10 +9927,10 @@ really_inline error_code json_structural_indexer::finish(dom_parser_implementati parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; // a valid JSON file cannot have zero structural indexes - we should have found something - if (unlikely(parser.n_structural_indexes == 0u)) { + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - if (unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { return UNEXPECTED_ERROR; } if (partial) { @@ -10000,18 +10000,18 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - really_inline int quote_index() { return trailing_zeroes(quote_bits); } - really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } uint32_t bs_bits; uint32_t quote_bits; }; // struct backslash_and_quote -really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 15 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); @@ -10068,8 +10068,8 @@ static const uint8_t escape_map[256] = { // dest will advance a variable amount (return via pointer) // return true if the unicode codepoint was valid // We work in little-endian then swap at write time -WARN_UNUSED -really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **dst_ptr) { // hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the @@ -10102,7 +10102,7 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, return offset > 0; } -WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +SIMDJSON_WARN_UNUSED simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { src++; while (1) { // Copy the next n bytes, and find the backslash and quote in them. @@ -10149,7 +10149,7 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst return nullptr; } -UNUSED WARN_UNUSED really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { +SIMDJSON_UNUSED SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { if (src[0] != '"') { return STRING_ERROR; } auto end = stringparsing::parse_string(src, current_string_buf_loc); if (!end) { return STRING_ERROR; } @@ -10173,7 +10173,7 @@ UNUSED WARN_UNUSED really_inline error_code parse_string_to_buffer(const uint8_t namespace { namespace SIMDJSON_IMPLEMENTATION { -static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); const __m128i mul_1_10 = @@ -10223,7 +10223,7 @@ namespace numberparsing { // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [FASTFLOAT_SMALLEST_POWER, // FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. -really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -10316,7 +10316,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // know that we have an exact computed value for the leading // 55 bits because any imprecision would play out as a +1, in // the worst case. - if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + if (simdjson_unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { uint64_t factor_mantissa_low = mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit @@ -10353,7 +10353,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // which we guard against. // If we have lots of trailing zeros, we may fall right between two // floating-point values. - if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + if (simdjson_unlikely((lower == 0) && ((upper & 0x1FF) == 0) && ((mantissa & 3) == 1))) { // if mantissa & 1 == 1 we might need to round up. // @@ -10389,7 +10389,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do mantissa &= ~(1ULL << 52); uint64_t real_exponent = c.exp - lz; // we have to check that real_exponent is in range, otherwise we bail out - if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + if (simdjson_unlikely((real_exponent < 1) || (real_exponent > 2046))) { return false; } mantissa |= real_exponent << 52; @@ -10430,7 +10430,7 @@ static bool parse_float_strtod(const uint8_t *ptr, double *outDouble) { // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -10446,7 +10446,7 @@ really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { } template -error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { +error_code slow_float_parsing(SIMDJSON_UNUSED const uint8_t * src, W writer) { double d; if (parse_float_strtod(src, &d)) { writer.append_double(d); @@ -10457,7 +10457,7 @@ error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { template NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -10467,7 +10467,7 @@ really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_really_inline error_code parse_decimal(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -10493,7 +10493,7 @@ really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const ui return SUCCESS; } -really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_really_inline error_code parse_exponent(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -10506,14 +10506,14 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. // Thus we *must* check for possible overflow before we negate exp_number. - // Performance notes: it may seem like combining the two "unlikely checks" below into - // a single unlikely path would be faster. The reasoning is sound, but the compiler may + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may // not oblige and may, in fact, generate two distinct paths in any case. It might be // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a likely branch, an unconclusive gain. + // instructions for a simdjson_likely branch, an unconclusive gain. // If there were no digits, it's an error. - if (unlikely(p == start_exp)) { + if (simdjson_unlikely(p == start_exp)) { return INVALID_NUMBER(src); } // We have a valid positive exponent in exp_number at this point, except that @@ -10521,7 +10521,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! - if (unlikely(p > start_exp+18)) { + if (simdjson_unlikely(p > start_exp+18)) { // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't @@ -10544,7 +10544,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u return SUCCESS; } -really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { +simdjson_really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -10556,12 +10556,12 @@ really_inline int significant_digits(const uint8_t * start_digits, int digit_cou } template -really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { +simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. // digit count is off by 1 because of the decimal (assuming there was one). - if (unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { + if (simdjson_unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -10578,10 +10578,10 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui writer.skip_double(); return error; } - // NOTE: it's weird that the unlikely() only wraps half the if, but it seems to get slower any other + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { + if (simdjson_unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { // this is almost never going to get called!!! // we start anew, going slowly!!! // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens @@ -10606,7 +10606,7 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui #ifdef SIMDJSON_SKIPNUMBERPARSING template -really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } @@ -10623,7 +10623,7 @@ really_inline error_code parse_number(const uint8_t *const, W &writer) { // // Our objective is accurate parsing (ULP of 0) at high speed. template -really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign @@ -10706,7 +10706,7 @@ really_inline error_code parse_number(const uint8_t *const src, W &writer) { // SAX functions namespace { // Parse any number from 0 to 18,446,744,073,709,551,615 -UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -10746,7 +10746,7 @@ UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * co // Parse any number from 0 to 18,446,744,073,709,551,615 // Call this version of the method if you regularly expect 8- or 16-digit numbers. -UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -10800,7 +10800,7 @@ UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // @@ -10849,7 +10849,7 @@ UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) return negative ? (~i+1) : i; } -UNUSED really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // @@ -10872,7 +10872,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // int64_t exponent = 0; bool overflow; - if (likely(*p == '.')) { + if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits @@ -10882,7 +10882,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Overflow check. 19 digits (minus the decimal) may be overflow. overflow = p-src-1 >= 19; - if (unlikely(overflow && leading_zero)) { + if (simdjson_unlikely(overflow && leading_zero)) { // Skip leading 0.00000 and see if it still overflows const uint8_t *start_digits = src + 2; while (*start_digits == '0') { start_digits++; } @@ -10914,7 +10914,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Assemble (or slow-parse) the float // double d; - if (likely(!overflow)) { + if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } if (!parse_float_strtod(src-negative, &d)) { @@ -10953,7 +10953,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static really_inline char printable_char(char c) { + static simdjson_really_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -10962,7 +10962,7 @@ namespace logger { } // Print the header and set up log_start - static really_inline void log_start() { + static simdjson_really_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -10973,7 +10973,7 @@ namespace logger { // Logs a single line of template - static really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -11049,14 +11049,14 @@ public: * - increment_count(iter) - each time a value is found in an array or object. */ template - WARN_UNUSED really_inline error_code walk_document(V &visitor) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -11065,7 +11065,7 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *peek() const noexcept; + simdjson_really_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -11073,56 +11073,56 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *advance() noexcept; + simdjson_really_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - really_inline size_t remaining_len() const noexcept; + simdjson_really_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - really_inline bool at_eof() const noexcept; + simdjson_really_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - really_inline bool at_beginning() const noexcept; - really_inline uint8_t last_structural() const noexcept; + simdjson_really_inline bool at_beginning() const noexcept; + simdjson_really_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_value(const char *type) const noexcept; + simdjson_really_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_start_value(const char *type) const noexcept; + simdjson_really_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_end_value(const char *type) const noexcept; + simdjson_really_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_error(const char *error) const noexcept; + simdjson_really_inline void log_error(const char *error) const noexcept; template - WARN_UNUSED really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - WARN_UNUSED really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -WARN_UNUSED really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -11173,7 +11173,7 @@ object_begin: } object_field: - if (unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { auto value = advance(); switch (*value) { @@ -11189,7 +11189,7 @@ object_continue: SIMDJSON_TRY( visitor.increment_count(*this) ); { auto key = advance(); - if (unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } goto object_field; @@ -11247,52 +11247,52 @@ document_end: } // walk_document() -really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -really_inline bool json_iterator::at_eof() const noexcept { +simdjson_really_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_really_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -11308,7 +11308,7 @@ WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visi } } template -WARN_UNUSED really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -11338,18 +11338,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - really_inline void append_s64(int64_t value) noexcept; + simdjson_really_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - really_inline void append_u64(uint64_t value) noexcept; + simdjson_really_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - really_inline void append_double(double value) noexcept; + simdjson_really_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -11357,24 +11357,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - really_inline void skip() noexcept; + simdjson_really_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - really_inline void skip_large_integer() noexcept; + simdjson_really_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - really_inline void skip_double() noexcept; + simdjson_really_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -11382,50 +11382,50 @@ private: * all 64 bits, such as double and uint64_t. */ template - really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -really_inline void tape_writer::append_double(double value) noexcept { +simdjson_really_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -really_inline void tape_writer::skip() noexcept { +simdjson_really_inline void tape_writer::skip() noexcept { next_tape_loc++; } -really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_really_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::skip_double() noexcept { +simdjson_really_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -11445,50 +11445,50 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -WARN_UNUSED -really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); return srcval ^ string_to_uint32(atom); } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -11506,40 +11506,40 @@ namespace stage2 { struct tape_builder { template - WARN_UNUSED static really_inline error_code parse_document( + SIMDJSON_WARN_UNUSED static simdjson_really_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - WARN_UNUSED really_inline error_code visit_document_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - WARN_UNUSED really_inline error_code visit_document_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - WARN_UNUSED really_inline error_code visit_array_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - WARN_UNUSED really_inline error_code visit_array_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - WARN_UNUSED really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - WARN_UNUSED really_inline error_code visit_object_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - WARN_UNUSED really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - WARN_UNUSED really_inline error_code visit_object_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - WARN_UNUSED really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - WARN_UNUSED really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -11548,22 +11548,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - WARN_UNUSED really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - WARN_UNUSED really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - WARN_UNUSED really_inline error_code increment_count(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -11571,18 +11571,18 @@ private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - really_inline tape_builder(dom::document &doc) noexcept; + simdjson_really_inline tape_builder(dom::document &doc) noexcept; - really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - really_inline void start_container(json_iterator &iter) noexcept; - WARN_UNUSED really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - WARN_UNUSED really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_really_inline void start_container(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -WARN_UNUSED really_inline error_code tape_builder::parse_document( +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -11591,56 +11591,56 @@ WARN_UNUSED really_inline error_code tape_builder::parse_document( return iter.walk_document(builder); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -WARN_UNUSED really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value, dst); @@ -11652,16 +11652,16 @@ WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &i return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -11684,42 +11684,42 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterat return error; } -WARN_UNUSED really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -11728,24 +11728,24 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_ite // private: -really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -WARN_UNUSED really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -11758,13 +11758,13 @@ WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator & return SUCCESS; } -really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -11788,36 +11788,36 @@ namespace { namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } return find_escaped_branchless(backslash); } } // namespace stage1 -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { +SIMDJSON_WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); } -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { this->buf = _buf; this->len = _len; return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); } -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { +SIMDJSON_WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return haswell::stage1::generic_validate_utf8(buf,len); } -WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, false); if (error) { return error; } return stage2(_doc); @@ -11882,7 +11882,7 @@ namespace SIMDJSON_IMPLEMENTATION { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. NO_SANITIZE_UNDEFINED -really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_really_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long ret; // Search the mask data from least significant bit (LSB) @@ -11895,12 +11895,12 @@ really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { return input_num & (input_num-1); } /* result might be undefined when input_num is zero */ -really_inline int leading_zeroes(uint64_t input_num) { +simdjson_really_inline int leading_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long leading_zero = 0; // Search the mask data from most significant bit (MSB) @@ -11915,17 +11915,17 @@ really_inline int leading_zeroes(uint64_t input_num) { } #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO -really_inline unsigned __int64 count_ones(uint64_t input_num) { +simdjson_really_inline unsigned __int64 count_ones(uint64_t input_num) { // note: we do not support legacy 32-bit Windows return __popcnt64(input_num);// Visual Studio wants two underscores } #else -really_inline long long int count_ones(uint64_t input_num) { +simdjson_really_inline long long int count_ones(uint64_t input_num) { return _popcnt64(input_num); } #endif -really_inline bool add_overflow(uint64_t value1, uint64_t value2, +simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return _addcarry_u64(0, value1, value2, @@ -11953,7 +11953,7 @@ namespace SIMDJSON_IMPLEMENTATION { // // For example, prefix_xor(00100100) == 00011100 // -really_inline uint64_t prefix_xor(const uint64_t bitmask) { +simdjson_really_inline uint64_t prefix_xor(const uint64_t bitmask) { // There should be no such thing with a processing supporting avx2 // but not clmul. __m128i all_ones = _mm_set1_epi8('\xFF'); @@ -11981,23 +11981,23 @@ namespace simd { __m128i value; // Zero constructor - really_inline base() : value{__m128i()} {} + simdjson_really_inline base() : value{__m128i()} {} // Conversion from SIMD register - really_inline base(const __m128i _value) : value(_value) {} + simdjson_really_inline base(const __m128i _value) : value(_value) {} // Conversion to SIMD register - really_inline operator const __m128i&() const { return this->value; } - really_inline operator __m128i&() { return this->value; } + simdjson_really_inline operator const __m128i&() const { return this->value; } + simdjson_really_inline operator __m128i&() { return this->value; } // Bit operations - really_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - really_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - really_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - really_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - really_inline Child& operator|=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast | other; return *this_cast; } - really_inline Child& operator&=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast & other; return *this_cast; } - really_inline Child& operator^=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_really_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdjson_really_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdjson_really_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdjson_really_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdjson_really_inline Child& operator|=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast | other; return *this_cast; } + simdjson_really_inline Child& operator&=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast & other; return *this_cast; } + simdjson_really_inline Child& operator^=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast ^ other; return *this_cast; } }; // Forward-declared so they can be used by splat and friends. @@ -12009,15 +12009,15 @@ namespace simd { typedef uint16_t bitmask_t; typedef uint32_t bitmask2_t; - really_inline base8() : base>() {} - really_inline base8(const __m128i _value) : base>(_value) {} + simdjson_really_inline base8() : base>() {} + simdjson_really_inline base8(const __m128i _value) : base>(_value) {} - really_inline Mask operator==(const simd8 other) const { return _mm_cmpeq_epi8(*this, other); } + simdjson_really_inline Mask operator==(const simd8 other) const { return _mm_cmpeq_epi8(*this, other); } static const int SIZE = sizeof(base>::value); template - really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { return _mm_alignr_epi8(*this, prev_chunk, 16 - N); } }; @@ -12025,27 +12025,27 @@ namespace simd { // SIMD byte mask type (returned by things like eq and gt) template<> struct simd8: base8 { - static really_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_really_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - really_inline simd8() : base8() {} - really_inline simd8(const __m128i _value) : base8(_value) {} + simdjson_really_inline simd8() : base8() {} + simdjson_really_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - really_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_really_inline simd8(bool _value) : base8(splat(_value)) {} - really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - really_inline bool any() const { return !_mm_testz_si128(*this, *this); } - really_inline simd8 operator~() const { return *this ^ true; } + simdjson_really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdjson_really_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdjson_really_inline simd8 operator~() const { return *this ^ true; } }; template struct base8_numeric: base8 { - static really_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static really_inline simd8 zero() { return _mm_setzero_si128(); } - static really_inline simd8 load(const T values[16]) { + static simdjson_really_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdjson_really_inline simd8 zero() { return _mm_setzero_si128(); } + static simdjson_really_inline simd8 load(const T values[16]) { return _mm_loadu_si128(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static really_inline simd8 repeat_16( + static simdjson_really_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -12055,24 +12055,24 @@ namespace simd { ); } - really_inline base8_numeric() : base8() {} - really_inline base8_numeric(const __m128i _value) : base8(_value) {} + simdjson_really_inline base8_numeric() : base8() {} + simdjson_really_inline base8_numeric(const __m128i _value) : base8(_value) {} // Store to array - really_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } + simdjson_really_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } // Override to distinguish from bool version - really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } // Addition/subtraction are the same for signed and unsigned - really_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - really_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *(simd8*)this; } - really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *(simd8*)this; } + simdjson_really_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdjson_really_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *(simd8*)this; } + simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *(simd8*)this; } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template - really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { return _mm_shuffle_epi8(lookup_table, *this); } @@ -12084,7 +12084,7 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - really_inline void compress(uint16_t mask, L * output) const { + simdjson_really_inline void compress(uint16_t mask, L * output) const { // this particular implementation was inspired by work done by @animetosho // we do it in two steps, first 8 bytes and then second 8 bytes uint8_t mask1 = uint8_t(mask); // least significant 8 bits @@ -12112,7 +12112,7 @@ namespace simd { } template - really_inline simd8 lookup_16( + simdjson_really_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -12129,14 +12129,14 @@ namespace simd { // Signed bytes template<> struct simd8 : base8_numeric { - really_inline simd8() : base8_numeric() {} - really_inline simd8(const __m128i _value) : base8_numeric(_value) {} + simdjson_really_inline simd8() : base8_numeric() {} + simdjson_really_inline simd8(const __m128i _value) : base8_numeric(_value) {} // Splat constructor - really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - really_inline simd8(const int8_t* values) : simd8(load(values)) {} + simdjson_really_inline simd8(const int8_t* values) : simd8(load(values)) {} // Member-by-member initialization - really_inline simd8( + simdjson_really_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(_mm_setr_epi8( @@ -12144,7 +12144,7 @@ namespace simd { v8, v9, v10,v11,v12,v13,v14,v15 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - really_inline static simd8 repeat_16( + simdjson_really_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -12155,23 +12155,23 @@ namespace simd { } // Order-sensitive comparisons - really_inline simd8 max(const simd8 other) const { return _mm_max_epi8(*this, other); } - really_inline simd8 min(const simd8 other) const { return _mm_min_epi8(*this, other); } - really_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - really_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + simdjson_really_inline simd8 max(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdjson_really_inline simd8 min(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdjson_really_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } }; // Unsigned bytes template<> struct simd8: base8_numeric { - really_inline simd8() : base8_numeric() {} - really_inline simd8(const __m128i _value) : base8_numeric(_value) {} + simdjson_really_inline simd8() : base8_numeric() {} + simdjson_really_inline simd8(const __m128i _value) : base8_numeric(_value) {} // Splat constructor - really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - really_inline simd8(const uint8_t* values) : simd8(load(values)) {} + simdjson_really_inline simd8(const uint8_t* values) : simd8(load(values)) {} // Member-by-member initialization - really_inline simd8( + simdjson_really_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(_mm_setr_epi8( @@ -12179,7 +12179,7 @@ namespace simd { v8, v9, v10,v11,v12,v13,v14,v15 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - really_inline static simd8 repeat_16( + simdjson_really_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -12190,39 +12190,39 @@ namespace simd { } // Saturated math - really_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - really_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + simdjson_really_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } // Order-specific operations - really_inline simd8 max(const simd8 other) const { return _mm_max_epu8(*this, other); } - really_inline simd8 min(const simd8 other) const { return _mm_min_epu8(*this, other); } + simdjson_really_inline simd8 max(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdjson_really_inline simd8 min(const simd8 other) const { return _mm_min_epu8(*this, other); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - really_inline simd8 operator<=(const simd8 other) const { return other.max(*this) == other; } - really_inline simd8 operator>=(const simd8 other) const { return other.min(*this) == other; } - really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - really_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_really_inline simd8 operator<=(const simd8 other) const { return other.max(*this) == other; } + simdjson_really_inline simd8 operator>=(const simd8 other) const { return other.min(*this) == other; } + simdjson_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } // Bit-specific operations - really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + simdjson_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdjson_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - really_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_really_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - really_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_really_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - really_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } + simdjson_really_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } }; template @@ -12235,28 +12235,28 @@ namespace simd { simd8x64& operator=(const simd8 other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - really_inline void store(T ptr[64]) const { + simdjson_really_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); this->chunks[2].store(ptr+sizeof(simd8)*2); this->chunks[3].store(ptr+sizeof(simd8)*3); } - really_inline simd8 reduce_or() const { + simdjson_really_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - really_inline void compress(uint64_t mask, T * output) const { + simdjson_really_inline void compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); } - really_inline uint64_t to_bitmask() const { + simdjson_really_inline uint64_t to_bitmask() const { uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); uint64_t r1 = this->chunks[1].to_bitmask(); uint64_t r2 = this->chunks[2].to_bitmask(); @@ -12264,7 +12264,7 @@ namespace simd { return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); } - really_inline simd8x64 bit_or(const T m) const { + simdjson_really_inline simd8x64 bit_or(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] | mask, @@ -12274,7 +12274,7 @@ namespace simd { ); } - really_inline uint64_t eq(const T m) const { + simdjson_really_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -12284,7 +12284,7 @@ namespace simd { ).to_bitmask(); } - really_inline uint64_t lteq(const T m) const { + simdjson_really_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -12333,17 +12333,17 @@ public: /** Document passed to stage 2 */ dom::document *doc{}; - really_inline dom_parser_implementation(); + simdjson_really_inline dom_parser_implementation(); dom_parser_implementation(const dom_parser_implementation &) = delete; dom_parser_implementation & operator=(const dom_parser_implementation &) = delete; - WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; - WARN_UNUSED error_code check_for_unclosed_array() noexcept; - WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; - WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; - WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; - WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; + SIMDJSON_WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, bool partial) noexcept final; + SIMDJSON_WARN_UNUSED error_code check_for_unclosed_array() noexcept; + SIMDJSON_WARN_UNUSED error_code stage2(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final; + SIMDJSON_WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; }; } // namespace SIMDJSON_IMPLEMENTATION @@ -12358,8 +12358,8 @@ namespace allocate { // // Allocates stage 1 internal state and outputs in the parser // -really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { - size_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7; +simdjson_really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) { + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; parser.structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); if (!parser.structural_indexes) { return MEMALLOC; } parser.structural_indexes[0] = 0; @@ -12381,7 +12381,7 @@ namespace allocate { // // Allocates stage 2 internal state and outputs in the parser // -really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { +simdjson_really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) { parser.open_containers.reset(new (std::nothrow) open_container[max_depth]); parser.is_array.reset(new (std::nothrow) bool[max_depth]); @@ -12400,17 +12400,17 @@ really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t namespace { namespace SIMDJSON_IMPLEMENTATION { -really_inline dom_parser_implementation::dom_parser_implementation() {} +simdjson_really_inline dom_parser_implementation::dom_parser_implementation() {} // Leaving these here so they can be inlined if so desired -WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { error_code err = stage1::allocate::set_capacity(*this, capacity); if (err) { _capacity = 0; return err; } _capacity = capacity; return SUCCESS; } -WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { error_code err = stage2::allocate::set_max_depth(*this, max_depth); if (err) { _max_depth = 0; return err; } _max_depth = max_depth; @@ -12427,7 +12427,7 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth namespace { namespace SIMDJSON_IMPLEMENTATION { -WARN_UNUSED error_code implementation::create_dom_parser_implementation( +SIMDJSON_WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, size_t max_depth, std::unique_ptr& dst @@ -12469,11 +12469,11 @@ namespace stage2 { // return non-zero if not a structural or whitespace char // zero otherwise -really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return structural_or_whitespace_negated[c]; } -really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { return structural_or_whitespace[c]; } @@ -12504,7 +12504,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -12536,10 +12536,10 @@ really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -12551,7 +12551,7 @@ static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { } #endif -really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -12584,17 +12584,17 @@ namespace SIMDJSON_IMPLEMENTATION { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); - really_inline uint64_t whitespace() const { return _whitespace; } - really_inline uint64_t op() const { return _op; } - really_inline uint64_t scalar() { return ~(op() | whitespace()); } + simdjson_really_inline uint64_t whitespace() const { return _whitespace; } + simdjson_really_inline uint64_t op() const { return _op; } + simdjson_really_inline uint64_t scalar() { return ~(op() | whitespace()); } uint64_t _whitespace; uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -12622,11 +12622,11 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(const simd8x64& input) { +simdjson_really_inline bool is_ascii(const simd8x64& input) { return input.reduce_or().is_ascii(); } -UNUSED really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { +SIMDJSON_UNUSED simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 @@ -12634,7 +12634,7 @@ UNUSED really_inline simd8 must_be_continuation(const simd8 prev1 return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { +simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. @@ -12651,7 +12651,7 @@ namespace utf8_validation { using namespace simd; - really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -12741,7 +12741,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -12754,7 +12754,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(const simd8 input) { + simdjson_really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -12778,7 +12778,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -12787,14 +12787,14 @@ using namespace simd; } // The only problem that can happen at EOF is that a multibyte character is too short. - really_inline void check_eof() { + simdjson_really_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - really_inline void check_next_input(const simd8x64& input) { - if(likely(is_ascii(input))) { + simdjson_really_inline void check_next_input(const simd8x64& input) { + if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. @@ -12815,7 +12815,7 @@ using namespace simd; } } - really_inline error_code errors() { + simdjson_really_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -12841,10 +12841,10 @@ namespace SIMDJSON_IMPLEMENTATION { template struct buf_block_reader { public: - really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - really_inline size_t block_index(); - really_inline bool has_full_block() const; - really_inline const uint8_t *full_block() const; + simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_really_inline size_t block_index(); + simdjson_really_inline bool has_full_block() const; + simdjson_really_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -12854,8 +12854,8 @@ public: * * @return the number of effective characters in the last block. */ - really_inline size_t get_remainder(uint8_t *dst) const; - really_inline void advance(); + simdjson_really_inline size_t get_remainder(uint8_t *dst) const; + simdjson_really_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -12864,7 +12864,7 @@ private: }; // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text_64(const uint8_t *text) { +SIMDJSON_UNUSED static char * format_input_text_64(const uint8_t *text) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); for (size_t i=0; i); i++) { buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); @@ -12874,7 +12874,7 @@ UNUSED static char * format_input_text_64(const uint8_t *text) { } // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text(const simd8x64& in) { +SIMDJSON_UNUSED static char * format_input_text(const simd8x64& in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); in.store((uint8_t*)buf); for (size_t i=0; i); i++) { @@ -12884,7 +12884,7 @@ UNUSED static char * format_input_text(const simd8x64& in) { return buf; } -UNUSED static char * format_mask(uint64_t mask) { +SIMDJSON_UNUSED static char * format_mask(uint64_t mask) { static char *buf = (char*)malloc(64 + 1); for (size_t i=0; i<64; i++) { buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; @@ -12894,30 +12894,30 @@ UNUSED static char * format_mask(uint64_t mask) { } template -really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } template -really_inline bool buf_block_reader::has_full_block() const { +simdjson_really_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { memset(dst, 0x20, STEP_SIZE); // memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. memcpy(dst, buf + idx, len - idx); return len - idx; } template -really_inline void buf_block_reader::advance() { +simdjson_really_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -12931,23 +12931,23 @@ namespace stage1 { struct json_string_block { // Escaped characters (characters following an escape() character) - really_inline uint64_t escaped() const { return _escaped; } + simdjson_really_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - really_inline uint64_t quote() const { return _quote; } + simdjson_really_inline uint64_t quote() const { return _quote; } // Start quotes of strings - really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -12962,13 +12962,13 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - really_inline json_string_block next(const simd::simd8x64& in); - really_inline error_code finish(bool streaming); + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_really_inline error_code finish(bool streaming); private: // Intended to be defined by the implementation - really_inline uint64_t find_escaped(uint64_t escape); - really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_really_inline uint64_t find_escaped(uint64_t escape); + simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -13003,7 +13003,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -13028,7 +13028,7 @@ really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t bac // // Backslash sequences outside of quotes will be detected in stage 2. // -really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -13058,7 +13058,7 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in); - really_inline error_code finish(bool streaming); + simdjson_really_inline json_block next(const simd::simd8x64& in); + simdjson_really_inline error_code finish(bool streaming); private: // Whether the last character of the previous iteration is part of a scalar token @@ -13180,13 +13180,13 @@ private: // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structurat characters json_character_block characters = json_character_block::classify(in); @@ -13209,7 +13209,7 @@ really_inline json_block json_scanner::next(const simd::simd8x64& in) { }; } -really_inline error_code json_scanner::finish(bool streaming) { +simdjson_really_inline error_code json_scanner::finish(bool streaming) { return string_scanner.finish(streaming); } @@ -13233,24 +13233,24 @@ public: static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; private: - really_inline json_minifier(uint8_t *_dst) + simdjson_really_inline json_minifier(uint8_t *_dst) : dst{_dst} {} template - really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - really_inline void next(const simd::simd8x64& in, json_block block); - really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_really_inline void next(const simd::simd8x64& in, json_block block); + simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { +simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { uint64_t mask = block.whitespace(); in.compress(mask, dst); dst += 64 - count_ones(mask); } -really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { *dst = '\0'; error_code error = scanner.finish(false); if (error) { dst_len = 0; return error; } @@ -13259,7 +13259,7 @@ really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_l } template<> -really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -13270,7 +13270,7 @@ really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_ } template<> -really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -13289,7 +13289,7 @@ error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, s // Index the last (remainder) block, padded with spaces uint8_t block[STEP_SIZE]; - if (likely(reader.get_remainder(block)) > 0) { + if (simdjson_likely(reader.get_remainder(block)) > 0) { minifier.step(block, reader); } @@ -13329,7 +13329,7 @@ namespace SIMDJSON_IMPLEMENTATION { * complete document, therefore the last json buffer location is the end of the * batch. */ -really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // TODO don't count separately, just figure out depth auto arr_cnt = 0; auto obj_cnt = 0; @@ -13382,14 +13382,14 @@ class bit_indexer { public: uint32_t *tail; - really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this - really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -13405,7 +13405,7 @@ public: // Do the next 8 all together (we hope in most cases it won't happen at all // and the branch is easily predicted). - if (unlikely(cnt > 8)) { + if (simdjson_unlikely(cnt > 8)) { for (int i=8; i<16; i++) { this->tail[i] = idx + trailing_zeroes(bits); bits = clear_lowest_bit(bits); @@ -13414,7 +13414,7 @@ public: // Most files don't have 16+ structurals per block, so we take several basically guaranteed // branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :) // or the start of a value ("abc" true 123) every four characters. - if (unlikely(cnt > 16)) { + if (simdjson_unlikely(cnt > 16)) { int i = 16; do { this->tail[i] = idx + trailing_zeroes(bits); @@ -13441,11 +13441,11 @@ public: static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, bool partial) noexcept; private: - really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); template - really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); - really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); + simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); + simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); json_scanner scanner{}; utf8_checker checker{}; @@ -13454,11 +13454,11 @@ private: uint64_t unescaped_chars_error = 0; }; -really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (unlikely(len < 3)) { +simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { + if (simdjson_unlikely(len < 3)) { switch (len) { case 2: if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left @@ -13496,7 +13496,7 @@ really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { // template error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, bool partial) noexcept { - if (unlikely(len > parser.capacity())) { return CAPACITY; } + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } if (partial) { len = trim_partial_utf8(buf, len); } buf_block_reader reader(buf, len); @@ -13509,14 +13509,14 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa // Take care of the last block (will always be there unless file is empty) uint8_t block[STEP_SIZE]; - if (unlikely(reader.get_remainder(block) == 0)) { return EMPTY; } + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return EMPTY; } indexer.step(block, reader); return indexer.finish(parser, reader.block_index(), len, partial); } template<> -really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -13527,14 +13527,14 @@ really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_ } template<> -really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { +simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -13542,12 +13542,12 @@ really_inline void json_structural_indexer::next(const simd::simd8x64& unescaped_chars_error |= block.non_quote_inside_string(unescaped); } -really_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial) { +simdjson_really_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial) { // Write out the final iteration's structurals indexer.write(uint32_t(idx-64), prev_structurals); error_code error = scanner.finish(partial); - if (unlikely(error != SUCCESS)) { return error; } + if (simdjson_unlikely(error != SUCCESS)) { return error; } if (unescaped_chars_error) { return UNESCAPED_CHARS; @@ -13573,10 +13573,10 @@ really_inline error_code json_structural_indexer::finish(dom_parser_implementati parser.structural_indexes[parser.n_structural_indexes + 2] = 0; parser.next_structural_index = 0; // a valid JSON file cannot have zero structural indexes - we should have found something - if (unlikely(parser.n_structural_indexes == 0u)) { + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - if (unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { return UNEXPECTED_ERROR; } if (partial) { @@ -13643,18 +13643,18 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - really_inline bool has_backslash() { return bs_bits != 0; } - really_inline int quote_index() { return trailing_zeroes(quote_bits); } - really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_really_inline bool has_backslash() { return bs_bits != 0; } + simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } uint32_t bs_bits; uint32_t quote_bits; }; // struct backslash_and_quote -really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); @@ -13713,8 +13713,8 @@ static const uint8_t escape_map[256] = { // dest will advance a variable amount (return via pointer) // return true if the unicode codepoint was valid // We work in little-endian then swap at write time -WARN_UNUSED -really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **dst_ptr) { // hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the @@ -13747,7 +13747,7 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, return offset > 0; } -WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +SIMDJSON_WARN_UNUSED simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { src++; while (1) { // Copy the next n bytes, and find the backslash and quote in them. @@ -13794,7 +13794,7 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst return nullptr; } -UNUSED WARN_UNUSED really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { +SIMDJSON_UNUSED SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { if (src[0] != '"') { return STRING_ERROR; } auto end = stringparsing::parse_string(src, current_string_buf_loc); if (!end) { return STRING_ERROR; } @@ -13818,7 +13818,7 @@ UNUSED WARN_UNUSED really_inline error_code parse_string_to_buffer(const uint8_t namespace { namespace SIMDJSON_IMPLEMENTATION { -static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); const __m128i mul_1_10 = @@ -13868,7 +13868,7 @@ namespace numberparsing { // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [FASTFLOAT_SMALLEST_POWER, // FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. -really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -13961,7 +13961,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // know that we have an exact computed value for the leading // 55 bits because any imprecision would play out as a +1, in // the worst case. - if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + if (simdjson_unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { uint64_t factor_mantissa_low = mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit @@ -13998,7 +13998,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do // which we guard against. // If we have lots of trailing zeros, we may fall right between two // floating-point values. - if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + if (simdjson_unlikely((lower == 0) && ((upper & 0x1FF) == 0) && ((mantissa & 3) == 1))) { // if mantissa & 1 == 1 we might need to round up. // @@ -14034,7 +14034,7 @@ really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, do mantissa &= ~(1ULL << 52); uint64_t real_exponent = c.exp - lz; // we have to check that real_exponent is in range, otherwise we bail out - if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + if (simdjson_unlikely((real_exponent < 1) || (real_exponent > 2046))) { return false; } mantissa |= real_exponent << 52; @@ -14075,7 +14075,7 @@ static bool parse_float_strtod(const uint8_t *ptr, double *outDouble) { // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -14091,7 +14091,7 @@ really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { } template -error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { +error_code slow_float_parsing(SIMDJSON_UNUSED const uint8_t * src, W writer) { double d; if (parse_float_strtod(src, &d)) { writer.append_double(d); @@ -14102,7 +14102,7 @@ error_code slow_float_parsing(UNUSED const uint8_t * src, W writer) { template NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -14112,7 +14112,7 @@ really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_really_inline error_code parse_decimal(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -14138,7 +14138,7 @@ really_inline error_code parse_decimal(UNUSED const uint8_t *const src, const ui return SUCCESS; } -really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_really_inline error_code parse_exponent(SIMDJSON_UNUSED const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -14151,14 +14151,14 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. // Thus we *must* check for possible overflow before we negate exp_number. - // Performance notes: it may seem like combining the two "unlikely checks" below into - // a single unlikely path would be faster. The reasoning is sound, but the compiler may + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may // not oblige and may, in fact, generate two distinct paths in any case. It might be // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a likely branch, an unconclusive gain. + // instructions for a simdjson_likely branch, an unconclusive gain. // If there were no digits, it's an error. - if (unlikely(p == start_exp)) { + if (simdjson_unlikely(p == start_exp)) { return INVALID_NUMBER(src); } // We have a valid positive exponent in exp_number at this point, except that @@ -14166,7 +14166,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! - if (unlikely(p > start_exp+18)) { + if (simdjson_unlikely(p > start_exp+18)) { // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't @@ -14189,7 +14189,7 @@ really_inline error_code parse_exponent(UNUSED const uint8_t *const src, const u return SUCCESS; } -really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { +simdjson_really_inline int significant_digits(const uint8_t * start_digits, int digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -14201,12 +14201,12 @@ really_inline int significant_digits(const uint8_t * start_digits, int digit_cou } template -really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { +simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, int digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. // digit count is off by 1 because of the decimal (assuming there was one). - if (unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { + if (simdjson_unlikely(digit_count-1 >= 19 && significant_digits(start_digits, digit_count) >= 19)) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -14223,10 +14223,10 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui writer.skip_double(); return error; } - // NOTE: it's weird that the unlikely() only wraps half the if, but it seems to get slower any other + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { + if (simdjson_unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || (exponent > FASTFLOAT_LARGEST_POWER)) { // this is almost never going to get called!!! // we start anew, going slowly!!! // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens @@ -14251,7 +14251,7 @@ really_inline error_code write_float(const uint8_t *const src, bool negative, ui #ifdef SIMDJSON_SKIPNUMBERPARSING template -really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } @@ -14268,7 +14268,7 @@ really_inline error_code parse_number(const uint8_t *const, W &writer) { // // Our objective is accurate parsing (ULP of 0) at high speed. template -really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign @@ -14351,7 +14351,7 @@ really_inline error_code parse_number(const uint8_t *const src, W &writer) { // SAX functions namespace { // Parse any number from 0 to 18,446,744,073,709,551,615 -UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -14391,7 +14391,7 @@ UNUSED really_inline simdjson_result parse_unsigned(const uint8_t * co // Parse any number from 0 to 18,446,744,073,709,551,615 // Call this version of the method if you regularly expect 8- or 16-digit numbers. -UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_large_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // @@ -14445,7 +14445,7 @@ UNUSED really_inline simdjson_result parse_large_unsigned(const uint8_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // @@ -14494,7 +14494,7 @@ UNUSED really_inline simdjson_result parse_integer(const uint8_t *src) return negative ? (~i+1) : i; } -UNUSED really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +SIMDJSON_UNUSED simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // @@ -14517,7 +14517,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // int64_t exponent = 0; bool overflow; - if (likely(*p == '.')) { + if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits @@ -14527,7 +14527,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Overflow check. 19 digits (minus the decimal) may be overflow. overflow = p-src-1 >= 19; - if (unlikely(overflow && leading_zero)) { + if (simdjson_unlikely(overflow && leading_zero)) { // Skip leading 0.00000 and see if it still overflows const uint8_t *start_digits = src + 2; while (*start_digits == '0') { start_digits++; } @@ -14559,7 +14559,7 @@ UNUSED really_inline simdjson_result parse_double(const uint8_t * src) n // Assemble (or slow-parse) the float // double d; - if (likely(!overflow)) { + if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } if (!parse_float_strtod(src-negative, &d)) { @@ -14598,7 +14598,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static really_inline char printable_char(char c) { + static simdjson_really_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -14607,7 +14607,7 @@ namespace logger { } // Print the header and set up log_start - static really_inline void log_start() { + static simdjson_really_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -14618,7 +14618,7 @@ namespace logger { // Logs a single line of template - static really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -14694,14 +14694,14 @@ public: * - increment_count(iter) - each time a value is found in an array or object. */ template - WARN_UNUSED really_inline error_code walk_document(V &visitor) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -14710,7 +14710,7 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *peek() const noexcept; + simdjson_really_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -14718,56 +14718,56 @@ public: * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - really_inline const uint8_t *advance() noexcept; + simdjson_really_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - really_inline size_t remaining_len() const noexcept; + simdjson_really_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - really_inline bool at_eof() const noexcept; + simdjson_really_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - really_inline bool at_beginning() const noexcept; - really_inline uint8_t last_structural() const noexcept; + simdjson_really_inline bool at_beginning() const noexcept; + simdjson_really_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_value(const char *type) const noexcept; + simdjson_really_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_start_value(const char *type) const noexcept; + simdjson_really_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_end_value(const char *type) const noexcept; + simdjson_really_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set ENABLE_LOGGING=true in logger.h to see logging. */ - really_inline void log_error(const char *error) const noexcept; + simdjson_really_inline void log_error(const char *error) const noexcept; template - WARN_UNUSED really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - WARN_UNUSED really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -WARN_UNUSED really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -14818,7 +14818,7 @@ object_begin: } object_field: - if (unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { auto value = advance(); switch (*value) { @@ -14834,7 +14834,7 @@ object_continue: SIMDJSON_TRY( visitor.increment_count(*this) ); { auto key = advance(); - if (unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } goto object_field; @@ -14892,52 +14892,52 @@ document_end: } // walk_document() -really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -really_inline bool json_iterator::at_eof() const noexcept { +simdjson_really_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_really_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -14953,7 +14953,7 @@ WARN_UNUSED really_inline error_code json_iterator::visit_root_primitive(V &visi } } template -WARN_UNUSED really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -14983,18 +14983,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - really_inline void append_s64(int64_t value) noexcept; + simdjson_really_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - really_inline void append_u64(uint64_t value) noexcept; + simdjson_really_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - really_inline void append_double(double value) noexcept; + simdjson_really_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -15002,24 +15002,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - really_inline void skip() noexcept; + simdjson_really_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - really_inline void skip_large_integer() noexcept; + simdjson_really_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - really_inline void skip_double() noexcept; + simdjson_really_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -15027,50 +15027,50 @@ private: * all 64 bits, such as double and uint64_t. */ template - really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -really_inline void tape_writer::append_double(double value) noexcept { +simdjson_really_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -really_inline void tape_writer::skip() noexcept { +simdjson_really_inline void tape_writer::skip() noexcept { next_tape_loc++; } -really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_really_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::skip_double() noexcept { +simdjson_really_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -15090,50 +15090,50 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -WARN_UNUSED -really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); return srcval ^ string_to_uint32(atom); } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; } -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; } -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +SIMDJSON_WARN_UNUSED +simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -15151,40 +15151,40 @@ namespace stage2 { struct tape_builder { template - WARN_UNUSED static really_inline error_code parse_document( + SIMDJSON_WARN_UNUSED static simdjson_really_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - WARN_UNUSED really_inline error_code visit_document_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - WARN_UNUSED really_inline error_code visit_document_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - WARN_UNUSED really_inline error_code visit_array_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - WARN_UNUSED really_inline error_code visit_array_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - WARN_UNUSED really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - WARN_UNUSED really_inline error_code visit_object_start(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - WARN_UNUSED really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - WARN_UNUSED really_inline error_code visit_object_end(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - WARN_UNUSED really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - WARN_UNUSED really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -15193,22 +15193,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - WARN_UNUSED really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - WARN_UNUSED really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - WARN_UNUSED really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - WARN_UNUSED really_inline error_code increment_count(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -15216,18 +15216,18 @@ private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - really_inline tape_builder(dom::document &doc) noexcept; + simdjson_really_inline tape_builder(dom::document &doc) noexcept; - really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - really_inline void start_container(json_iterator &iter) noexcept; - WARN_UNUSED really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - WARN_UNUSED really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_really_inline void start_container(json_iterator &iter) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -WARN_UNUSED really_inline error_code tape_builder::parse_document( +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -15236,56 +15236,56 @@ WARN_UNUSED really_inline error_code tape_builder::parse_document( return iter.walk_document(builder); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -WARN_UNUSED really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -WARN_UNUSED really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -WARN_UNUSED really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value, dst); @@ -15297,16 +15297,16 @@ WARN_UNUSED really_inline error_code tape_builder::visit_string(json_iterator &i return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -WARN_UNUSED really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -15329,42 +15329,42 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_number(json_iterat return error; } -WARN_UNUSED really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -15373,24 +15373,24 @@ WARN_UNUSED really_inline error_code tape_builder::visit_root_null_atom(json_ite // private: -really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -WARN_UNUSED really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -15403,13 +15403,13 @@ WARN_UNUSED really_inline error_code tape_builder::end_container(json_iterator & return SUCCESS; } -really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -15434,36 +15434,36 @@ namespace { namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } return find_escaped_branchless(backslash); } } // namespace stage1 -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { +SIMDJSON_WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); } -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { this->buf = _buf; this->len = _len; return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); } -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { +SIMDJSON_WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return westmere::stage1::generic_validate_utf8(buf,len); } -WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } -WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { +SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, false); if (error) { return error; } return stage2(_doc); diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index f00db9aa..51f5897c 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Tue Aug 18 17:35:47 PDT 2020. Do not edit! */ +/* auto-generated on Wed 19 Aug 2020 17:23:29 EDT. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -229,12 +229,6 @@ use a 64-bit target such as x64 or 64-bit ARM.") #endif -#if SIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT -// No matter what happened, we undefine SIMDJSON_THREADS_ENABLED and so disable threads. -#undef SIMDJSON_THREADS_ENABLED -#endif - - #if defined(__clang__) #define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) #elif defined(__GNUC__) @@ -311,34 +305,34 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024; #if defined(__GNUC__) // Marks a block with a name so that MCA analysis can see it. - #define BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name); - #define END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name); - #define DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name); + #define SIMDJSON_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name); + #define SIMDJSON_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name); + #define SIMDJSON_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name); #else - #define BEGIN_DEBUG_BLOCK(name) - #define END_DEBUG_BLOCK(name) - #define DEBUG_BLOCK(name, block) + #define SIMDJSON_BEGIN_DEBUG_BLOCK(name) + #define SIMDJSON_END_DEBUG_BLOCK(name) + #define SIMDJSON_DEBUG_BLOCK(name, block) #endif // Align to N-byte boundary -#define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) -#define ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) +#define SIMDJSON_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) +#define SIMDJSON_ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) -#define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) +#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) - #define really_inline __forceinline - #define never_inline __declspec(noinline) + #define simdjson_really_inline __forceinline + #define simdjson_never_inline __declspec(noinline) - #define UNUSED - #define WARN_UNUSED + #define SIMDJSON_UNUSED + #define SIMDJSON_WARN_UNUSED - #ifndef likely - #define likely(x) x + #ifndef simdjson_likely + #define simdjson_likely(x) x #endif - #ifndef unlikely - #define unlikely(x) x + #ifndef simdjson_unlikely + #define simdjson_unlikely(x) x #endif #define SIMDJSON_PUSH_DISABLE_WARNINGS __pragma(warning( push )) @@ -362,17 +356,17 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024; #else // SIMDJSON_REGULAR_VISUAL_STUDIO - #define really_inline inline __attribute__((always_inline)) - #define never_inline inline __attribute__((noinline)) + #define simdjson_really_inline inline __attribute__((always_inline)) + #define simdjson_never_inline inline __attribute__((noinline)) - #define UNUSED __attribute__((unused)) - #define WARN_UNUSED __attribute__((warn_unused_result)) + #define SIMDJSON_UNUSED __attribute__((unused)) + #define SIMDJSON_WARN_UNUSED __attribute__((warn_unused_result)) - #ifndef likely - #define likely(x) __builtin_expect(!!(x), 1) + #ifndef simdjson_likely + #define simdjson_likely(x) __builtin_expect(!!(x), 1) #endif - #ifndef unlikely - #define unlikely(x) __builtin_expect(!!(x), 0) + #ifndef simdjson_unlikely + #define simdjson_unlikely(x) __builtin_expect(!!(x), 0) #endif #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") @@ -2013,7 +2007,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION 0.4.6 +#define SIMDJSON_VERSION 0.5.0 namespace simdjson { enum { @@ -2024,11 +2018,11 @@ enum { /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 4, + SIMDJSON_VERSION_MINOR = 5, /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 6 + SIMDJSON_VERSION_REVISION = 0 }; } // namespace simdjson @@ -2137,22 +2131,22 @@ struct simdjson_result_base : public std::pair { /** * Create a new empty result with error = UNINITIALIZED. */ - really_inline simdjson_result_base() noexcept; + simdjson_really_inline simdjson_result_base() noexcept; /** * Create a new error result. */ - really_inline simdjson_result_base(error_code error) noexcept; + simdjson_really_inline simdjson_result_base(error_code error) noexcept; /** * Create a new successful result. */ - really_inline simdjson_result_base(T &&value) noexcept; + simdjson_really_inline simdjson_result_base(T &&value) noexcept; /** * Create a new result with both things (use if you don't want to branch when creating the result). */ - really_inline simdjson_result_base(T &&value, error_code error) noexcept; + simdjson_really_inline simdjson_result_base(T &&value, error_code error) noexcept; /** * Move the value and the error to the provided variables. @@ -2160,19 +2154,19 @@ struct simdjson_result_base : public std::pair { * @param value The variable to assign the value to. May not be set if there is an error. * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ - really_inline void tie(T &value, error_code &error) && noexcept; + simdjson_really_inline void tie(T &value, error_code &error) && noexcept; /** * Move the value to the provided variable. * * @param value The variable to assign the value to. May not be set if there is an error. */ - really_inline error_code get(T &value) && noexcept; + simdjson_really_inline error_code get(T &value) && noexcept; /** * The error. */ - really_inline error_code error() const noexcept; + simdjson_really_inline error_code error() const noexcept; #if SIMDJSON_EXCEPTIONS @@ -2181,21 +2175,21 @@ struct simdjson_result_base : public std::pair { * * @throw simdjson_error if there was an error. */ - really_inline T& value() noexcept(false); + simdjson_really_inline T& value() noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - really_inline T&& take_value() && noexcept(false); + simdjson_really_inline T&& take_value() && noexcept(false); /** * Cast to the value (will throw on error). * * @throw simdjson_error if there was an error. */ - really_inline operator T&&() && noexcept(false); + simdjson_really_inline operator T&&() && noexcept(false); #endif // SIMDJSON_EXCEPTIONS }; // struct simdjson_result_base @@ -2212,19 +2206,19 @@ struct simdjson_result : public internal::simdjson_result_base { /** * @private Create a new empty result with error = UNINITIALIZED. */ - really_inline simdjson_result() noexcept; + simdjson_really_inline simdjson_result() noexcept; /** * @private Create a new error result. */ - really_inline simdjson_result(T &&value) noexcept; + simdjson_really_inline simdjson_result(T &&value) noexcept; /** * @private Create a new successful result. */ - really_inline simdjson_result(error_code error_code) noexcept; + simdjson_really_inline simdjson_result(error_code error_code) noexcept; /** * @private Create a new result with both things (use if you don't want to branch when creating the result). */ - really_inline simdjson_result(T &&value, error_code error) noexcept; + simdjson_really_inline simdjson_result(T &&value, error_code error) noexcept; /** * Move the value and the error to the provided variables. @@ -2232,19 +2226,19 @@ struct simdjson_result : public internal::simdjson_result_base { * @param value The variable to assign the value to. May not be set if there is an error. * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ - really_inline void tie(T &value, error_code &error) && noexcept; + simdjson_really_inline void tie(T &value, error_code &error) && noexcept; /** * Move the value to the provided variable. * * @param value The variable to assign the value to. May not be set if there is an error. */ - WARN_UNUSED really_inline error_code get(T &value) && noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code get(T &value) && noexcept; /** * The error. */ - really_inline error_code error() const noexcept; + simdjson_really_inline error_code error() const noexcept; #if SIMDJSON_EXCEPTIONS @@ -2253,21 +2247,21 @@ struct simdjson_result : public internal::simdjson_result_base { * * @throw simdjson_error if there was an error. */ - really_inline T& value() noexcept(false); + simdjson_really_inline T& value() noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - really_inline T&& take_value() && noexcept(false); + simdjson_really_inline T&& take_value() && noexcept(false); /** * Cast to the value (will throw on error). * * @throw simdjson_error if there was an error. */ - really_inline operator T&&() && noexcept(false); + simdjson_really_inline operator T&&() && noexcept(false); #endif // SIMDJSON_EXCEPTIONS }; // struct simdjson_result @@ -2484,7 +2478,7 @@ public: * @param len The length of the json document. * @return The error code, or SUCCESS if there was no error. */ - WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept = 0; + SIMDJSON_WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept = 0; /** * @private For internal implementation use @@ -2500,7 +2494,7 @@ public: * @param streaming Whether this is being called by parser::parse_many. * @return The error code, or SUCCESS if there was no error. */ - WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, bool streaming) noexcept = 0; + SIMDJSON_WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, bool streaming) noexcept = 0; /** * @private For internal implementation use @@ -2514,7 +2508,7 @@ public: * @param doc The document to output to. * @return The error code, or SUCCESS if there was no error. */ - WARN_UNUSED virtual error_code stage2(dom::document &doc) noexcept = 0; + SIMDJSON_WARN_UNUSED virtual error_code stage2(dom::document &doc) noexcept = 0; /** * @private For internal implementation use @@ -2527,7 +2521,7 @@ public: * @param doc The document to output to. * @return The error code, SUCCESS if there was no error, or EMPTY if all documents have been parsed. */ - WARN_UNUSED virtual error_code stage2_next(dom::document &doc) noexcept = 0; + SIMDJSON_WARN_UNUSED virtual error_code stage2_next(dom::document &doc) noexcept = 0; /** * Change the capacity of this parser. @@ -2568,14 +2562,14 @@ public: * * @return Current capacity, in bytes. */ - really_inline size_t capacity() const noexcept; + simdjson_really_inline size_t capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. * * @return Maximum depth, in bytes. */ - really_inline size_t max_depth() const noexcept; + simdjson_really_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -2585,7 +2579,7 @@ public: * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. * @return The error, if there is one. */ - WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth) noexcept; + SIMDJSON_WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth) noexcept; protected: /** @@ -2603,15 +2597,15 @@ protected: size_t _max_depth{0}; }; // class dom_parser_implementation -really_inline size_t dom_parser_implementation::capacity() const noexcept { +simdjson_really_inline size_t dom_parser_implementation::capacity() const noexcept { return _capacity; } -really_inline size_t dom_parser_implementation::max_depth() const noexcept { +simdjson_really_inline size_t dom_parser_implementation::max_depth() const noexcept { return _max_depth; } -WARN_UNUSED +SIMDJSON_WARN_UNUSED inline error_code dom_parser_implementation::allocate(size_t capacity, size_t max_depth) noexcept { if (this->max_depth() != max_depth) { error_code err = set_max_depth(max_depth); @@ -2642,7 +2636,7 @@ namespace simdjson { * @param len the length of the string in bytes. * @return true if the string is valid UTF-8. */ -WARN_UNUSED bool validate_utf8(const char * buf, size_t len) noexcept; +SIMDJSON_WARN_UNUSED bool validate_utf8(const char * buf, size_t len) noexcept; /** @@ -2651,7 +2645,7 @@ WARN_UNUSED bool validate_utf8(const char * buf, size_t len) noexcept; * @param sv the string_view to validate. * @return true if the string is valid UTF-8. */ -really_inline WARN_UNUSED bool validate_utf8(const std::string_view sv) noexcept { +simdjson_really_inline SIMDJSON_WARN_UNUSED bool validate_utf8(const std::string_view sv) noexcept { return validate_utf8(sv.data(), sv.size()); } @@ -2661,7 +2655,7 @@ really_inline WARN_UNUSED bool validate_utf8(const std::string_view sv) noexcept * @param p the string to validate. * @return true if the string is valid UTF-8. */ -really_inline WARN_UNUSED bool validate_utf8(const std::string& s) noexcept { +simdjson_really_inline SIMDJSON_WARN_UNUSED bool validate_utf8(const std::string& s) noexcept { return validate_utf8(s.data(), s.size()); } @@ -2737,7 +2731,7 @@ public: * @param dst_len the number of bytes written. Output only. * @return the error code, or SUCCESS if there was no error. */ - WARN_UNUSED virtual error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept = 0; + SIMDJSON_WARN_UNUSED virtual error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept = 0; /** @@ -2749,11 +2743,11 @@ public: * @param len the length of the string in bytes. * @return true if and only if the string is valid UTF-8. */ - WARN_UNUSED virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0; + SIMDJSON_WARN_UNUSED virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0; protected: /** @private Construct an implementation with the given name and description. For subclasses. */ - really_inline implementation( + simdjson_really_inline implementation( std::string_view name, std::string_view description, uint32_t required_instruction_sets @@ -2791,7 +2785,7 @@ namespace internal { class available_implementation_list { public: /** Get the list of available implementations compiled into simdjson */ - really_inline available_implementation_list() {} + simdjson_really_inline available_implementation_list() {} /** Number of implementations */ size_t size() const noexcept; /** STL const begin() iterator */ @@ -2924,25 +2918,25 @@ constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF; */ class tape_ref { public: - really_inline tape_ref() noexcept; - really_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; + simdjson_really_inline tape_ref() noexcept; + simdjson_really_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; inline size_t after_element() const noexcept; - really_inline tape_type tape_ref_type() const noexcept; - really_inline uint64_t tape_value() const noexcept; - really_inline bool is_double() const noexcept; - really_inline bool is_int64() const noexcept; - really_inline bool is_uint64() const noexcept; - really_inline bool is_false() const noexcept; - really_inline bool is_true() const noexcept; - really_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null. - really_inline uint32_t matching_brace_index() const noexcept; - really_inline uint32_t scope_count() const noexcept; + simdjson_really_inline tape_type tape_ref_type() const noexcept; + simdjson_really_inline uint64_t tape_value() const noexcept; + simdjson_really_inline bool is_double() const noexcept; + simdjson_really_inline bool is_int64() const noexcept; + simdjson_really_inline bool is_uint64() const noexcept; + simdjson_really_inline bool is_false() const noexcept; + simdjson_really_inline bool is_true() const noexcept; + simdjson_really_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null. + simdjson_really_inline uint32_t matching_brace_index() const noexcept; + simdjson_really_inline uint32_t scope_count() const noexcept; template - really_inline T next_tape_value() const noexcept; - really_inline uint32_t get_string_length() const noexcept; - really_inline const char * get_c_str() const noexcept; + simdjson_really_inline T next_tape_value() const noexcept; + simdjson_really_inline uint32_t get_string_length() const noexcept; + simdjson_really_inline const char * get_c_str() const noexcept; inline std::string_view get_string_view() const noexcept; - really_inline bool is_document_root() const noexcept; + simdjson_really_inline bool is_document_root() const noexcept; /** The document this element references. */ const dom::document *doc; @@ -2981,7 +2975,7 @@ namespace simdjson { * @param dst_len the number of bytes written. Output only. * @return the error code, or SUCCESS if there was no error. */ -WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept; +SIMDJSON_WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept; /** * Minifies a JSON element or document, printing the smallest possible valid JSON. @@ -3045,7 +3039,7 @@ class element; class array { public: /** Create a new, invalid array */ - really_inline array() noexcept; + simdjson_really_inline array() noexcept; class iterator { public: @@ -3086,7 +3080,7 @@ public: iterator(const iterator&) noexcept = default; iterator& operator=(const iterator&) noexcept = default; private: - really_inline iterator(const internal::tape_ref &tape) noexcept; + simdjson_really_inline iterator(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; friend class array; }; @@ -3146,7 +3140,7 @@ public: inline simdjson_result at(size_t index) const noexcept; private: - really_inline array(const internal::tape_ref &tape) noexcept; + simdjson_really_inline array(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; friend class element; friend struct simdjson_result; @@ -3171,9 +3165,9 @@ inline std::ostream& operator<<(std::ostream& out, const array &value); template<> struct simdjson_result : public internal::simdjson_result_base { public: - really_inline simdjson_result() noexcept; ///< @private - really_inline simdjson_result(dom::array value) noexcept; ///< @private - really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_really_inline simdjson_result() noexcept; ///< @private + simdjson_really_inline simdjson_result(dom::array value) noexcept; ///< @private + simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; inline simdjson_result at(size_t index) const noexcept; @@ -3342,20 +3336,20 @@ public: * to allocate an initial capacity, call allocate() after constructing the parser. * Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process). */ - really_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; + simdjson_really_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; /** * Take another parser's buffers and state. * * @param other The parser to take. Its capacity is zeroed. */ - really_inline parser(parser &&other) noexcept; + simdjson_really_inline parser(parser &&other) noexcept; parser(const parser &) = delete; ///< @private Disallow copying /** * Take another parser's buffers and state. * * @param other The parser to take. Its capacity is zeroed. */ - really_inline parser &operator=(parser &&other) noexcept; + simdjson_really_inline parser &operator=(parser &&other) noexcept; parser &operator=(const parser &) = delete; ///< @private Disallow copying /** Deallocate the JSON parser. */ @@ -3425,17 +3419,17 @@ public: inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - really_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; - really_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete; + simdjson_really_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; + simdjson_really_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete; /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - really_inline simdjson_result parse(const std::string &s) & noexcept; - really_inline simdjson_result parse(const std::string &s) && =delete; + simdjson_really_inline simdjson_result parse(const std::string &s) & noexcept; + simdjson_really_inline simdjson_result parse(const std::string &s) && =delete; /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - really_inline simdjson_result parse(const padded_string &s) & noexcept; - really_inline simdjson_result parse(const padded_string &s) && =delete; + simdjson_really_inline simdjson_result parse(const padded_string &s) & noexcept; + simdjson_really_inline simdjson_result parse(const padded_string &s) && =delete; /** @private We do not want to allow implicit conversion from C string to std::string. */ - really_inline simdjson_result parse(const char *buf) noexcept = delete; + simdjson_really_inline simdjson_result parse(const char *buf) noexcept = delete; /** * Load a file containing many JSON documents. @@ -3587,7 +3581,7 @@ public: * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. * @return The error, if there is one. */ - WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; + SIMDJSON_WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; /** * @private deprecated because it returns bool instead of error_code, which is our standard for @@ -3601,14 +3595,14 @@ public: * @return true if successful, false if allocation failed. */ [[deprecated("Use allocate() instead.")]] - WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; + SIMDJSON_WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; /** * The largest document this parser can support without reallocating. * * @return Current capacity, in bytes. */ - really_inline size_t capacity() const noexcept; + simdjson_really_inline size_t capacity() const noexcept; /** * The largest document this parser can automatically support. @@ -3617,14 +3611,14 @@ public: * * @return Maximum capacity, in bytes. */ - really_inline size_t max_capacity() const noexcept; + simdjson_really_inline size_t max_capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. * * @return Maximum depth, in bytes. */ - really_inline size_t max_depth() const noexcept; + simdjson_really_inline size_t max_depth() const noexcept; /** * Set max_capacity. This is the largest document this parser can automatically support. @@ -3636,7 +3630,7 @@ public: * * @param max_capacity The new maximum capacity, in bytes. */ - really_inline void set_max_capacity(size_t max_capacity) noexcept; + simdjson_really_inline void set_max_capacity(size_t max_capacity) noexcept; #ifdef SIMDJSON_THREADS_ENABLED /** @@ -3811,13 +3805,13 @@ public: * error = parser.parse_many(json).get(docs); * ``` */ - really_inline document_stream() noexcept; + simdjson_really_inline document_stream() noexcept; /** Move one document_stream to another. */ - really_inline document_stream(document_stream &&other) noexcept = default; + simdjson_really_inline document_stream(document_stream &&other) noexcept = default; /** Move one document_stream to another. */ - really_inline document_stream &operator=(document_stream &&other) noexcept = default; + simdjson_really_inline document_stream &operator=(document_stream &&other) noexcept = default; - really_inline ~document_stream() noexcept; + simdjson_really_inline ~document_stream() noexcept; /** * An iterator through a forward-only stream of documents. @@ -3827,7 +3821,7 @@ public: /** * Get the current document (or error). */ - really_inline simdjson_result operator*() noexcept; + simdjson_really_inline simdjson_result operator*() noexcept; /** * Advance to the next document. */ @@ -3836,7 +3830,7 @@ public: * Check if we're at the end yet. * @param other the end iterator to compare to. */ - really_inline bool operator!=(const iterator &other) const noexcept; + simdjson_really_inline bool operator!=(const iterator &other) const noexcept; /** * @private * @@ -3852,7 +3846,7 @@ public: * may change in future versions of simdjson: we find the API somewhat * awkward and we would like to offer something friendlier. */ - really_inline size_t current_index() const noexcept; + simdjson_really_inline size_t current_index() const noexcept; /** * @private * @@ -3872,10 +3866,10 @@ public: * may change in future versions of simdjson: we find the API somewhat * awkward and we would like to offer something friendlier. */ - really_inline std::string_view source() const noexcept; + simdjson_really_inline std::string_view source() const noexcept; private: - really_inline iterator(document_stream &s, bool finished) noexcept; + simdjson_really_inline iterator(document_stream &s, bool finished) noexcept; /** The document_stream we're iterating through. */ document_stream& stream; /** Whether we're finished or not. */ @@ -3886,11 +3880,11 @@ public: /** * Start iterating the documents in the stream. */ - really_inline iterator begin() noexcept; + simdjson_really_inline iterator begin() noexcept; /** * The end of the stream, for iterator comparison purposes. */ - really_inline iterator end() noexcept; + simdjson_really_inline iterator end() noexcept; private: @@ -3906,7 +3900,7 @@ private: * @param len is the length of the raw byte buffer in bytes * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) */ - really_inline document_stream( + simdjson_really_inline document_stream( dom::parser &parser, const uint8_t *buf, size_t len, @@ -4001,18 +3995,18 @@ private: template<> struct simdjson_result : public internal::simdjson_result_base { public: - really_inline simdjson_result() noexcept; ///< @private - really_inline simdjson_result(error_code error) noexcept; ///< @private - really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private + simdjson_really_inline simdjson_result() noexcept; ///< @private + simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private #if SIMDJSON_EXCEPTIONS - really_inline dom::document_stream::iterator begin() noexcept(false); - really_inline dom::document_stream::iterator end() noexcept(false); + simdjson_really_inline dom::document_stream::iterator begin() noexcept(false); + simdjson_really_inline dom::document_stream::iterator end() noexcept(false); #else // SIMDJSON_EXCEPTIONS [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - really_inline dom::document_stream::iterator begin() noexcept; + simdjson_really_inline dom::document_stream::iterator begin() noexcept; [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - really_inline dom::document_stream::iterator end() noexcept; + simdjson_really_inline dom::document_stream::iterator end() noexcept; #endif // SIMDJSON_EXCEPTIONS }; // struct simdjson_result @@ -4057,10 +4051,10 @@ enum class element_type { class element { public: /** Create a new, invalid element. */ - really_inline element() noexcept; + simdjson_really_inline element() noexcept; /** The type of this element. */ - really_inline element_type type() const noexcept; + simdjson_really_inline element_type type() const noexcept; /** * Cast this element to an array. @@ -4226,7 +4220,7 @@ public: * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object */ template - really_inline bool is() const noexcept; + simdjson_really_inline bool is() const noexcept; /** * Get the value as the provided type (T). @@ -4263,7 +4257,7 @@ public: * @returns The error that occurred, or SUCCESS if there was no error. */ template - WARN_UNUSED really_inline error_code get(T &value) const noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code get(T &value) const noexcept; /** * Get the value as the provided type (T), setting error if it's not the given type. @@ -4485,7 +4479,7 @@ public: inline bool dump_raw_tape(std::ostream &out) const noexcept; private: - really_inline element(const internal::tape_ref &tape) noexcept; + simdjson_really_inline element(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; friend class document; friend class object; @@ -4521,58 +4515,58 @@ inline std::ostream& operator<<(std::ostream& out, element_type type); template<> struct simdjson_result : public internal::simdjson_result_base { public: - really_inline simdjson_result() noexcept; ///< @private - really_inline simdjson_result(dom::element &&value) noexcept; ///< @private - really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_really_inline simdjson_result() noexcept; ///< @private + simdjson_really_inline simdjson_result(dom::element &&value) noexcept; ///< @private + simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - really_inline simdjson_result type() const noexcept; + simdjson_really_inline simdjson_result type() const noexcept; template - really_inline bool is() const noexcept; + simdjson_really_inline bool is() const noexcept; template - really_inline simdjson_result get() const noexcept; + simdjson_really_inline simdjson_result get() const noexcept; template - WARN_UNUSED really_inline error_code get(T &value) const noexcept; + SIMDJSON_WARN_UNUSED simdjson_really_inline error_code get(T &value) const noexcept; - really_inline simdjson_result get_array() const noexcept; - really_inline simdjson_result get_object() const noexcept; - really_inline simdjson_result get_c_str() const noexcept; - really_inline simdjson_result get_string_length() const noexcept; - really_inline simdjson_result get_string() const noexcept; - really_inline simdjson_result get_int64() const noexcept; - really_inline simdjson_result get_uint64() const noexcept; - really_inline simdjson_result get_double() const noexcept; - really_inline simdjson_result get_bool() const noexcept; + simdjson_really_inline simdjson_result get_array() const noexcept; + simdjson_really_inline simdjson_result get_object() const noexcept; + simdjson_really_inline simdjson_result get_c_str() const noexcept; + simdjson_really_inline simdjson_result get_string_length() const noexcept; + simdjson_really_inline simdjson_result get_string() const noexcept; + simdjson_really_inline simdjson_result get_int64() const noexcept; + simdjson_really_inline simdjson_result get_uint64() const noexcept; + simdjson_really_inline simdjson_result get_double() const noexcept; + simdjson_really_inline simdjson_result get_bool() const noexcept; - really_inline bool is_array() const noexcept; - really_inline bool is_object() const noexcept; - really_inline bool is_string() const noexcept; - really_inline bool is_int64() const noexcept; - really_inline bool is_uint64() const noexcept; - really_inline bool is_double() const noexcept; - really_inline bool is_bool() const noexcept; - really_inline bool is_null() const noexcept; + simdjson_really_inline bool is_array() const noexcept; + simdjson_really_inline bool is_object() const noexcept; + simdjson_really_inline bool is_string() const noexcept; + simdjson_really_inline bool is_int64() const noexcept; + simdjson_really_inline bool is_uint64() const noexcept; + simdjson_really_inline bool is_double() const noexcept; + simdjson_really_inline bool is_bool() const noexcept; + simdjson_really_inline bool is_null() const noexcept; - really_inline simdjson_result operator[](std::string_view key) const noexcept; - really_inline simdjson_result operator[](const char *key) const noexcept; - really_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; + simdjson_really_inline simdjson_result operator[](std::string_view key) const noexcept; + simdjson_really_inline simdjson_result operator[](const char *key) const noexcept; + simdjson_really_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] - really_inline simdjson_result at(const std::string_view json_pointer) const noexcept; - really_inline simdjson_result at(size_t index) const noexcept; - really_inline simdjson_result at_key(std::string_view key) const noexcept; - really_inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; + simdjson_really_inline simdjson_result at(const std::string_view json_pointer) const noexcept; + simdjson_really_inline simdjson_result at(size_t index) const noexcept; + simdjson_really_inline simdjson_result at_key(std::string_view key) const noexcept; + simdjson_really_inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; #if SIMDJSON_EXCEPTIONS - really_inline operator bool() const noexcept(false); - really_inline explicit operator const char*() const noexcept(false); - really_inline operator std::string_view() const noexcept(false); - really_inline operator uint64_t() const noexcept(false); - really_inline operator int64_t() const noexcept(false); - really_inline operator double() const noexcept(false); - really_inline operator dom::array() const noexcept(false); - really_inline operator dom::object() const noexcept(false); + simdjson_really_inline operator bool() const noexcept(false); + simdjson_really_inline explicit operator const char*() const noexcept(false); + simdjson_really_inline operator std::string_view() const noexcept(false); + simdjson_really_inline operator uint64_t() const noexcept(false); + simdjson_really_inline operator int64_t() const noexcept(false); + simdjson_really_inline operator double() const noexcept(false); + simdjson_really_inline operator dom::array() const noexcept(false); + simdjson_really_inline operator dom::object() const noexcept(false); - really_inline dom::array::iterator begin() const noexcept(false); - really_inline dom::array::iterator end() const noexcept(false); + simdjson_really_inline dom::array::iterator begin() const noexcept(false); + simdjson_really_inline dom::array::iterator end() const noexcept(false); #endif // SIMDJSON_EXCEPTIONS }; @@ -4588,7 +4582,7 @@ public: * underlying output stream, that error will be propagated (simdjson_error will not be * thrown). */ -really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false); +simdjson_really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false); #endif } // namespace simdjson @@ -4614,7 +4608,7 @@ class key_value_pair; class object { public: /** Create a new, invalid object */ - really_inline object() noexcept; + simdjson_really_inline object() noexcept; class iterator { public: @@ -4684,7 +4678,7 @@ public: iterator(const iterator&) noexcept = default; iterator& operator=(const iterator&) noexcept = default; private: - really_inline iterator(const internal::tape_ref &tape) noexcept; + simdjson_really_inline iterator(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; @@ -4798,7 +4792,7 @@ public: inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; private: - really_inline object(const internal::tape_ref &tape) noexcept; + simdjson_really_inline object(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; @@ -4819,7 +4813,7 @@ public: element value; private: - really_inline key_value_pair(std::string_view _key, element _value) noexcept; + simdjson_really_inline key_value_pair(std::string_view _key, element _value) noexcept; friend class object; }; @@ -4850,9 +4844,9 @@ inline std::ostream& operator<<(std::ostream& out, const key_value_pair &value); template<> struct simdjson_result : public internal::simdjson_result_base { public: - really_inline simdjson_result() noexcept; ///< @private - really_inline simdjson_result(dom::object value) noexcept; ///< @private - really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_really_inline simdjson_result() noexcept; ///< @private + simdjson_really_inline simdjson_result(dom::object value) noexcept; ///< @private + simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private inline simdjson_result operator[](std::string_view key) const noexcept; inline simdjson_result operator[](const char *key) const noexcept; @@ -5000,7 +4994,7 @@ inline int json_parse(const padded_string &s, dom::parser &parser) noexcept { } [[deprecated("Use parser.parse() instead")]] -WARN_UNUSED inline dom::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept { +SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept { dom::parser parser; error_code code = parser.parse(buf, len, realloc_if_needed).error(); // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid @@ -5012,7 +5006,7 @@ WARN_UNUSED inline dom::parser build_parsed_json(const uint8_t *buf, size_t len, return parser; } [[deprecated("Use parser.parse() instead")]] -WARN_UNUSED inline dom::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept { +SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept { dom::parser parser; error_code code = parser.parse(buf, len, realloc_if_needed).error(); // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid @@ -5024,7 +5018,7 @@ WARN_UNUSED inline dom::parser build_parsed_json(const char *buf, size_t len, bo return parser; } [[deprecated("Use parser.parse() instead")]] -WARN_UNUSED inline dom::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept { +SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept { dom::parser parser; error_code code = parser.parse(s.data(), s.length(), realloc_if_needed).error(); // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid @@ -5036,7 +5030,7 @@ WARN_UNUSED inline dom::parser build_parsed_json(const std::string &s, bool real return parser; } [[deprecated("Use parser.parse() instead")]] -WARN_UNUSED inline dom::parser build_parsed_json(const padded_string &s) noexcept { +SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const padded_string &s) noexcept { dom::parser parser; error_code code = parser.parse(s).error(); // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid @@ -5395,7 +5389,7 @@ public: /* end file include/simdjson/internal/jsonformatutils.h */ // Inline functions -/* begin file include/simdjson/inline/array.h */ +/* begin file include/simdjson/dom/array-inl.h */ #ifndef SIMDJSON_INLINE_ARRAY_H #define SIMDJSON_INLINE_ARRAY_H @@ -5408,11 +5402,11 @@ namespace simdjson { // // simdjson_result inline implementation // -really_inline simdjson_result::simdjson_result() noexcept +simdjson_really_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} -really_inline simdjson_result::simdjson_result(dom::array value) noexcept +simdjson_really_inline simdjson_result::simdjson_result(dom::array value) noexcept : internal::simdjson_result_base(std::forward(value)) {} -really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} #if SIMDJSON_EXCEPTIONS @@ -5446,8 +5440,8 @@ namespace dom { // // array inline implementation // -really_inline array::array() noexcept : tape{} {} -really_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} +simdjson_really_inline array::array() noexcept : tape{} {} +simdjson_really_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} inline array::iterator array::begin() const noexcept { return internal::tape_ref(tape.doc, tape.json_index + 1); } @@ -5510,7 +5504,7 @@ inline simdjson_result array::at(size_t index) const noexcept { // // array::iterator inline implementation // -really_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_really_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element array::iterator::operator*() const noexcept { return element(tape); } @@ -5577,7 +5571,7 @@ inline std::ostream& operator<<(std::ostream& out, const simdjson_result inline implementation // -really_inline simdjson_result::simdjson_result() noexcept +simdjson_really_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} -really_inline simdjson_result::simdjson_result(dom::element &&value) noexcept +simdjson_really_inline simdjson_result::simdjson_result(dom::element &&value) noexcept : internal::simdjson_result_base(std::forward(value)) {} -really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} inline simdjson_result simdjson_result::type() const noexcept { if (error()) { return error(); } @@ -5601,148 +5595,148 @@ inline simdjson_result simdjson_result::type() } template -really_inline bool simdjson_result::is() const noexcept { +simdjson_really_inline bool simdjson_result::is() const noexcept { return !error() && first.is(); } template -really_inline simdjson_result simdjson_result::get() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get() const noexcept { if (error()) { return error(); } return first.get(); } template -WARN_UNUSED really_inline error_code simdjson_result::get(T &value) const noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code simdjson_result::get(T &value) const noexcept { if (error()) { return error(); } return first.get(value); } -really_inline simdjson_result simdjson_result::get_array() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_array() const noexcept { if (error()) { return error(); } return first.get_array(); } -really_inline simdjson_result simdjson_result::get_object() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_object() const noexcept { if (error()) { return error(); } return first.get_object(); } -really_inline simdjson_result simdjson_result::get_c_str() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_c_str() const noexcept { if (error()) { return error(); } return first.get_c_str(); } -really_inline simdjson_result simdjson_result::get_string_length() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_string_length() const noexcept { if (error()) { return error(); } return first.get_string_length(); } -really_inline simdjson_result simdjson_result::get_string() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_string() const noexcept { if (error()) { return error(); } return first.get_string(); } -really_inline simdjson_result simdjson_result::get_int64() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_int64() const noexcept { if (error()) { return error(); } return first.get_int64(); } -really_inline simdjson_result simdjson_result::get_uint64() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_uint64() const noexcept { if (error()) { return error(); } return first.get_uint64(); } -really_inline simdjson_result simdjson_result::get_double() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_double() const noexcept { if (error()) { return error(); } return first.get_double(); } -really_inline simdjson_result simdjson_result::get_bool() const noexcept { +simdjson_really_inline simdjson_result simdjson_result::get_bool() const noexcept { if (error()) { return error(); } return first.get_bool(); } -really_inline bool simdjson_result::is_array() const noexcept { +simdjson_really_inline bool simdjson_result::is_array() const noexcept { return !error() && first.is_array(); } -really_inline bool simdjson_result::is_object() const noexcept { +simdjson_really_inline bool simdjson_result::is_object() const noexcept { return !error() && first.is_object(); } -really_inline bool simdjson_result::is_string() const noexcept { +simdjson_really_inline bool simdjson_result::is_string() const noexcept { return !error() && first.is_string(); } -really_inline bool simdjson_result::is_int64() const noexcept { +simdjson_really_inline bool simdjson_result::is_int64() const noexcept { return !error() && first.is_int64(); } -really_inline bool simdjson_result::is_uint64() const noexcept { +simdjson_really_inline bool simdjson_result::is_uint64() const noexcept { return !error() && first.is_uint64(); } -really_inline bool simdjson_result::is_double() const noexcept { +simdjson_really_inline bool simdjson_result::is_double() const noexcept { return !error() && first.is_double(); } -really_inline bool simdjson_result::is_bool() const noexcept { +simdjson_really_inline bool simdjson_result::is_bool() const noexcept { return !error() && first.is_bool(); } -really_inline bool simdjson_result::is_null() const noexcept { +simdjson_really_inline bool simdjson_result::is_null() const noexcept { return !error() && first.is_null(); } -really_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { +simdjson_really_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { if (error()) { return error(); } return first[key]; } -really_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { +simdjson_really_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { if (error()) { return error(); } return first[key]; } -really_inline simdjson_result simdjson_result::at_pointer(const std::string_view json_pointer) const noexcept { +simdjson_really_inline simdjson_result simdjson_result::at_pointer(const std::string_view json_pointer) const noexcept { if (error()) { return error(); } return first.at_pointer(json_pointer); } [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] -really_inline simdjson_result simdjson_result::at(const std::string_view json_pointer) const noexcept { +simdjson_really_inline simdjson_result simdjson_result::at(const std::string_view json_pointer) const noexcept { SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_DEPRECATED_WARNING if (error()) { return error(); } return first.at(json_pointer); SIMDJSON_POP_DISABLE_WARNINGS } -really_inline simdjson_result simdjson_result::at(size_t index) const noexcept { +simdjson_really_inline simdjson_result simdjson_result::at(size_t index) const noexcept { if (error()) { return error(); } return first.at(index); } -really_inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { +simdjson_really_inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { if (error()) { return error(); } return first.at_key(key); } -really_inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { +simdjson_really_inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { if (error()) { return error(); } return first.at_key_case_insensitive(key); } #if SIMDJSON_EXCEPTIONS -really_inline simdjson_result::operator bool() const noexcept(false) { +simdjson_really_inline simdjson_result::operator bool() const noexcept(false) { return get(); } -really_inline simdjson_result::operator const char *() const noexcept(false) { +simdjson_really_inline simdjson_result::operator const char *() const noexcept(false) { return get(); } -really_inline simdjson_result::operator std::string_view() const noexcept(false) { +simdjson_really_inline simdjson_result::operator std::string_view() const noexcept(false) { return get(); } -really_inline simdjson_result::operator uint64_t() const noexcept(false) { +simdjson_really_inline simdjson_result::operator uint64_t() const noexcept(false) { return get(); } -really_inline simdjson_result::operator int64_t() const noexcept(false) { +simdjson_really_inline simdjson_result::operator int64_t() const noexcept(false) { return get(); } -really_inline simdjson_result::operator double() const noexcept(false) { +simdjson_really_inline simdjson_result::operator double() const noexcept(false) { return get(); } -really_inline simdjson_result::operator dom::array() const noexcept(false) { +simdjson_really_inline simdjson_result::operator dom::array() const noexcept(false) { return get(); } -really_inline simdjson_result::operator dom::object() const noexcept(false) { +simdjson_really_inline simdjson_result::operator dom::object() const noexcept(false) { return get(); } -really_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { +simdjson_really_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.begin(); } -really_inline dom::array::iterator simdjson_result::end() const noexcept(false) { +simdjson_really_inline dom::array::iterator simdjson_result::end() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.end(); } @@ -5754,8 +5748,8 @@ namespace dom { // // element inline implementation // -really_inline element::element() noexcept : tape{} {} -really_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_really_inline element::element() noexcept : tape{} {} +simdjson_really_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element_type element::type() const noexcept { auto tape_type = tape.tape_ref_type(); @@ -5797,7 +5791,7 @@ inline simdjson_result element::get_string() const noexcept { } } inline simdjson_result element::get_uint64() const noexcept { - if(unlikely(!tape.is_uint64())) { // branch rarely taken + if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken if(tape.is_int64()) { int64_t result = tape.next_tape_value(); if (result < 0) { @@ -5810,7 +5804,7 @@ inline simdjson_result element::get_uint64() const noexcept { return tape.next_tape_value(); } inline simdjson_result element::get_int64() const noexcept { - if(unlikely(!tape.is_int64())) { // branch rarely taken + if(simdjson_unlikely(!tape.is_int64())) { // branch rarely taken if(tape.is_uint64()) { uint64_t result = tape.next_tape_value(); // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std @@ -5833,7 +5827,7 @@ inline simdjson_result element::get_double() const noexcept { // We can expect get to refer to a double type almost all the time. // It is important to craft the code accordingly so that the compiler can use this // information. (This could also be solved with profile-guided optimization.) - if(unlikely(!tape.is_double())) { // branch rarely taken + if(simdjson_unlikely(!tape.is_double())) { // branch rarely taken if(tape.is_uint64()) { return double(tape.next_tape_value()); } else if(tape.is_int64()) { @@ -5862,18 +5856,18 @@ inline simdjson_result element::get_object() const noexcept { } template -WARN_UNUSED really_inline error_code element::get(T &value) const noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code element::get(T &value) const noexcept { return get().get(value); } // An element-specific version prevents recursion with simdjson_result::get(value) template<> -WARN_UNUSED really_inline error_code element::get(element &value) const noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code element::get(element &value) const noexcept { value = element(tape); return SUCCESS; } template -really_inline bool element::is() const noexcept { +simdjson_really_inline bool element::is() const noexcept { auto result = get(); return !result.error(); } @@ -6018,7 +6012,7 @@ inline std::ostream& minifier::print(std::ostream& out) { case tape_type::START_ARRAY: { // If we're too deep, we need to recurse to go deeper. depth++; - if (unlikely(depth >= MAX_DEPTH)) { + if (simdjson_unlikely(depth >= MAX_DEPTH)) { out << minify(dom::array(iter)); iter.json_index = iter.matching_brace_index() - 1; // Jump to the ] depth--; @@ -6045,7 +6039,7 @@ inline std::ostream& minifier::print(std::ostream& out) { case tape_type::START_OBJECT: { // If we're too deep, we need to recurse to go deeper. depth++; - if (unlikely(depth >= MAX_DEPTH)) { + if (simdjson_unlikely(depth >= MAX_DEPTH)) { out << minify(dom::object(iter)); iter.json_index = iter.matching_brace_index() - 1; // Jump to the } depth--; @@ -6119,12 +6113,12 @@ inline std::ostream& minifier::print(std::ostream& out) { #if SIMDJSON_EXCEPTIONS template<> -really_inline std::ostream& minifier>::print(std::ostream& out) { +simdjson_really_inline std::ostream& minifier>::print(std::ostream& out) { if (value.error()) { throw simdjson_error(value.error()); } return out << minify(value.first); } -really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false) { +simdjson_really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false) { return out << minify>(value); } #endif @@ -6132,7 +6126,7 @@ really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result< } // namespace simdjson #endif // SIMDJSON_INLINE_ELEMENT_H -/* end file include/simdjson/inline/element.h */ +/* end file include/simdjson/dom/element-inl.h */ #if defined(__cpp_lib_ranges) static_assert(std::ranges::view); @@ -6144,8 +6138,8 @@ static_assert(std::ranges::sized_range document_stream::iterator::operator*() noexcept { +simdjson_really_inline simdjson_result document_stream::iterator::operator*() noexcept { // Once we have yielded any errors, we're finished. if (stream.error) { finished = true; return stream.error; } return stream.parser->doc.root(); } -really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { +simdjson_really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { stream.next(); // If that was the last document, we're finished. if (stream.error == EMPTY) { finished = true; } return *this; } -really_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { +simdjson_really_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } @@ -6304,11 +6298,11 @@ inline void document_stream::start() noexcept { next(); } -really_inline size_t document_stream::iterator::current_index() const noexcept { +simdjson_really_inline size_t document_stream::iterator::current_index() const noexcept { return stream.doc_index; } -really_inline std::string_view document_stream::iterator::source() const noexcept { +simdjson_really_inline std::string_view document_stream::iterator::source() const noexcept { size_t next_doc_index = stream.batch_start + stream.parser->implementation->structural_indexes[stream.parser->implementation->next_structural_index]; return std::string_view(reinterpret_cast(stream.buf) + current_index(), next_doc_index - current_index() - 1); } @@ -6386,31 +6380,31 @@ inline void document_stream::start_stage1_thread() noexcept { } // namespace dom -really_inline simdjson_result::simdjson_result() noexcept +simdjson_really_inline simdjson_result::simdjson_result() noexcept : simdjson_result_base() { } -really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept : simdjson_result_base(error) { } -really_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept +simdjson_really_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept : simdjson_result_base(std::forward(value)) { } #if SIMDJSON_EXCEPTIONS -really_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { +simdjson_really_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.begin(); } -really_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { +simdjson_really_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.end(); } #else // SIMDJSON_EXCEPTIONS -really_inline dom::document_stream::iterator simdjson_result::begin() noexcept { +simdjson_really_inline dom::document_stream::iterator simdjson_result::begin() noexcept { first.error = error(); return first.begin(); } -really_inline dom::document_stream::iterator simdjson_result::end() noexcept { +simdjson_really_inline dom::document_stream::iterator simdjson_result::end() noexcept { first.error = error(); return first.end(); } @@ -6418,8 +6412,8 @@ really_inline dom::document_stream::iterator simdjson_result -really_inline void simdjson_result_base::tie(T &value, error_code &error) && noexcept { +simdjson_really_inline void simdjson_result_base::tie(T &value, error_code &error) && noexcept { // on the clang compiler that comes with current macOS (Apple clang version 11.0.0), // tie(width, error) = size["w"].get(); // fails with "error: no viable overloaded '='"" @@ -6613,49 +6607,49 @@ really_inline void simdjson_result_base::tie(T &value, error_code &error) && } template -WARN_UNUSED really_inline error_code simdjson_result_base::get(T &value) && noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code simdjson_result_base::get(T &value) && noexcept { error_code error; std::forward>(*this).tie(value, error); return error; } template -really_inline error_code simdjson_result_base::error() const noexcept { +simdjson_really_inline error_code simdjson_result_base::error() const noexcept { return this->second; } #if SIMDJSON_EXCEPTIONS template -really_inline T& simdjson_result_base::value() noexcept(false) { +simdjson_really_inline T& simdjson_result_base::value() noexcept(false) { if (error()) { throw simdjson_error(error()); } return this->first; } template -really_inline T&& simdjson_result_base::take_value() && noexcept(false) { +simdjson_really_inline T&& simdjson_result_base::take_value() && noexcept(false) { if (error()) { throw simdjson_error(error()); } return std::forward(this->first); } template -really_inline simdjson_result_base::operator T&&() && noexcept(false) { +simdjson_really_inline simdjson_result_base::operator T&&() && noexcept(false) { return std::forward>(*this).take_value(); } #endif // SIMDJSON_EXCEPTIONS template -really_inline simdjson_result_base::simdjson_result_base(T &&value, error_code error) noexcept +simdjson_really_inline simdjson_result_base::simdjson_result_base(T &&value, error_code error) noexcept : std::pair(std::forward(value), error) {} template -really_inline simdjson_result_base::simdjson_result_base(error_code error) noexcept +simdjson_really_inline simdjson_result_base::simdjson_result_base(error_code error) noexcept : simdjson_result_base(T{}, error) {} template -really_inline simdjson_result_base::simdjson_result_base(T &&value) noexcept +simdjson_really_inline simdjson_result_base::simdjson_result_base(T &&value) noexcept : simdjson_result_base(std::forward(value), SUCCESS) {} template -really_inline simdjson_result_base::simdjson_result_base() noexcept +simdjson_really_inline simdjson_result_base::simdjson_result_base() noexcept : simdjson_result_base(T{}, UNINITIALIZED) {} } // namespace internal @@ -6665,57 +6659,57 @@ really_inline simdjson_result_base::simdjson_result_base() noexcept /// template -really_inline void simdjson_result::tie(T &value, error_code &error) && noexcept { +simdjson_really_inline void simdjson_result::tie(T &value, error_code &error) && noexcept { std::forward>(*this).tie(value, error); } template -WARN_UNUSED really_inline error_code simdjson_result::get(T &value) && noexcept { +SIMDJSON_WARN_UNUSED simdjson_really_inline error_code simdjson_result::get(T &value) && noexcept { return std::forward>(*this).get(value); } template -really_inline error_code simdjson_result::error() const noexcept { +simdjson_really_inline error_code simdjson_result::error() const noexcept { return internal::simdjson_result_base::error(); } #if SIMDJSON_EXCEPTIONS template -really_inline T& simdjson_result::value() noexcept(false) { +simdjson_really_inline T& simdjson_result::value() noexcept(false) { return internal::simdjson_result_base::value(); } template -really_inline T&& simdjson_result::take_value() && noexcept(false) { +simdjson_really_inline T&& simdjson_result::take_value() && noexcept(false) { return std::forward>(*this).take_value(); } template -really_inline simdjson_result::operator T&&() && noexcept(false) { +simdjson_really_inline simdjson_result::operator T&&() && noexcept(false) { return std::forward>(*this).take_value(); } #endif // SIMDJSON_EXCEPTIONS template -really_inline simdjson_result::simdjson_result(T &&value, error_code error) noexcept +simdjson_really_inline simdjson_result::simdjson_result(T &&value, error_code error) noexcept : internal::simdjson_result_base(std::forward(value), error) {} template -really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} template -really_inline simdjson_result::simdjson_result(T &&value) noexcept +simdjson_really_inline simdjson_result::simdjson_result(T &&value) noexcept : internal::simdjson_result_base(std::forward(value)) {} template -really_inline simdjson_result::simdjson_result() noexcept +simdjson_really_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} } // namespace simdjson #endif // SIMDJSON_INLINE_ERROR_H -/* end file include/simdjson/inline/error.h */ -/* begin file include/simdjson/inline/object.h */ +/* end file include/simdjson/error-inl.h */ +/* begin file include/simdjson/dom/object-inl.h */ #ifndef SIMDJSON_INLINE_OBJECT_H #define SIMDJSON_INLINE_OBJECT_H @@ -6727,11 +6721,11 @@ namespace simdjson { // // simdjson_result inline implementation // -really_inline simdjson_result::simdjson_result() noexcept +simdjson_really_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} -really_inline simdjson_result::simdjson_result(dom::object value) noexcept +simdjson_really_inline simdjson_result::simdjson_result(dom::object value) noexcept : internal::simdjson_result_base(std::forward(value)) {} -really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { @@ -6777,8 +6771,8 @@ namespace dom { // // object inline implementation // -really_inline object::object() noexcept : tape{} {} -really_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_really_inline object::object() noexcept : tape{} {} +simdjson_really_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline object::iterator object::begin() const noexcept { return internal::tape_ref(tape.doc, tape.json_index + 1); } @@ -6866,7 +6860,7 @@ inline simdjson_result object::at_key_case_insensitive(std::string_view // // object::iterator inline implementation // -really_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_really_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline const key_value_pair object::iterator::operator*() const noexcept { return key_value_pair(key(), value()); } @@ -7006,8 +7000,8 @@ static_assert(std::ranges::sized_range padded_string::load(const std::string &fil } // namespace simdjson #endif // SIMDJSON_INLINE_PADDED_STRING_H -/* end file include/simdjson/inline/padded_string.h */ -/* begin file include/simdjson/inline/parsedjson_iterator.h */ +/* end file include/simdjson/padded_string-inl.h */ +/* begin file include/simdjson/dom/parsedjson_iterator-inl.h */ #ifndef SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H #define SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H @@ -7166,7 +7160,7 @@ SIMDJSON_DISABLE_DEPRECATED_WARNING // Because of template weirdness, the actual class definition is inline in the document class -WARN_UNUSED bool dom::parser::Iterator::is_ok() const { +SIMDJSON_WARN_UNUSED bool dom::parser::Iterator::is_ok() const { return location < tape_length; } @@ -7637,8 +7631,8 @@ SIMDJSON_POP_DISABLE_WARNINGS } // namespace simdjson #endif // SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H -/* end file include/simdjson/inline/parsedjson_iterator.h */ -/* begin file include/simdjson/inline/parser.h */ +/* end file include/simdjson/dom/parsedjson_iterator-inl.h */ +/* begin file include/simdjson/dom/parser-inl.h */ #ifndef SIMDJSON_INLINE_PARSER_H #define SIMDJSON_INLINE_PARSER_H @@ -7651,12 +7645,12 @@ namespace dom { // // parser inline implementation // -really_inline parser::parser(size_t max_capacity) noexcept +simdjson_really_inline parser::parser(size_t max_capacity) noexcept : _max_capacity{max_capacity}, loaded_bytes(nullptr) { } -really_inline parser::parser(parser &&other) noexcept = default; -really_inline parser &parser::operator=(parser &&other) noexcept = default; +simdjson_really_inline parser::parser(parser &&other) noexcept = default; +simdjson_really_inline parser &parser::operator=(parser &&other) noexcept = default; inline bool parser::is_valid() const noexcept { return valid; } inline int parser::get_error_code() const noexcept { return error; } @@ -7741,13 +7735,13 @@ inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bo return doc.root(); } -really_inline simdjson_result parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept { +simdjson_really_inline simdjson_result parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept { return parse((const uint8_t *)buf, len, realloc_if_needed); } -really_inline simdjson_result parser::parse(const std::string &s) & noexcept { +simdjson_really_inline simdjson_result parser::parse(const std::string &s) & noexcept { return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } -really_inline simdjson_result parser::parse(const padded_string &s) & noexcept { +simdjson_really_inline simdjson_result parser::parse(const padded_string &s) & noexcept { return parse(s.data(), s.length(), false); } @@ -7764,17 +7758,17 @@ inline simdjson_result parser::parse_many(const padded_string & return parse_many(s.data(), s.length(), batch_size); } -really_inline size_t parser::capacity() const noexcept { +simdjson_really_inline size_t parser::capacity() const noexcept { return implementation ? implementation->capacity() : 0; } -really_inline size_t parser::max_capacity() const noexcept { +simdjson_really_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } -really_inline size_t parser::max_depth() const noexcept { +simdjson_really_inline size_t parser::max_depth() const noexcept { return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH; } -WARN_UNUSED +SIMDJSON_WARN_UNUSED inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { // // Reallocate implementation and document if needed @@ -7800,7 +7794,7 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { return SUCCESS; } -WARN_UNUSED +SIMDJSON_WARN_UNUSED inline bool parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept { return !allocate(capacity, max_depth); } @@ -7809,7 +7803,7 @@ inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept { // If we don't have enough capacity, (try to) automatically bump it. // If the document was taken, reallocate that too. // Both in one if statement to minimize unlikely branching. - if (unlikely(capacity() < desired_capacity || !doc.tape)) { + if (simdjson_unlikely(capacity() < desired_capacity || !doc.tape)) { if (desired_capacity > max_capacity()) { return error = CAPACITY; } @@ -7819,7 +7813,7 @@ inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept { return SUCCESS; } -really_inline void parser::set_max_capacity(size_t max_capacity) noexcept { +simdjson_really_inline void parser::set_max_capacity(size_t max_capacity) noexcept { _max_capacity = max_capacity; } @@ -7827,8 +7821,8 @@ really_inline void parser::set_max_capacity(size_t max_capacity) noexcept { } // namespace simdjson #endif // SIMDJSON_INLINE_PARSER_H -/* end file include/simdjson/inline/parser.h */ -/* begin file include/simdjson/inline/tape_ref.h */ +/* end file include/simdjson/dom/parser-inl.h */ +/* begin file include/simdjson/internal/tape_ref-inl.h */ #ifndef SIMDJSON_INLINE_TAPE_REF_H #define SIMDJSON_INLINE_TAPE_REF_H @@ -7840,11 +7834,11 @@ namespace internal { // // tape_ref inline implementation // -really_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} -really_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} +simdjson_really_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} +simdjson_really_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} -really_inline bool tape_ref::is_document_root() const noexcept { +simdjson_really_inline bool tape_ref::is_document_root() const noexcept { return json_index == 1; // should we ever change the structure of the tape, this should get updated. } @@ -7852,27 +7846,27 @@ really_inline bool tape_ref::is_document_root() const noexcept { // to check the type by doing a word-to-word comparison instead of extracting the // most significant 8 bits. -really_inline bool tape_ref::is_double() const noexcept { +simdjson_really_inline bool tape_ref::is_double() const noexcept { constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56; return doc->tape[json_index] == tape_double; } -really_inline bool tape_ref::is_int64() const noexcept { +simdjson_really_inline bool tape_ref::is_int64() const noexcept { constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56; return doc->tape[json_index] == tape_int64; } -really_inline bool tape_ref::is_uint64() const noexcept { +simdjson_really_inline bool tape_ref::is_uint64() const noexcept { constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56; return doc->tape[json_index] == tape_uint64; } -really_inline bool tape_ref::is_false() const noexcept { +simdjson_really_inline bool tape_ref::is_false() const noexcept { constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56; return doc->tape[json_index] == tape_false; } -really_inline bool tape_ref::is_true() const noexcept { +simdjson_really_inline bool tape_ref::is_true() const noexcept { constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56; return doc->tape[json_index] == tape_true; } -really_inline bool tape_ref::is_null_on_tape() const noexcept { +simdjson_really_inline bool tape_ref::is_null_on_tape() const noexcept { constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56; return doc->tape[json_index] == tape_null; } @@ -7890,21 +7884,21 @@ inline size_t tape_ref::after_element() const noexcept { return json_index + 1; } } -really_inline tape_type tape_ref::tape_ref_type() const noexcept { +simdjson_really_inline tape_type tape_ref::tape_ref_type() const noexcept { return static_cast(doc->tape[json_index] >> 56); } -really_inline uint64_t internal::tape_ref::tape_value() const noexcept { +simdjson_really_inline uint64_t internal::tape_ref::tape_value() const noexcept { return doc->tape[json_index] & internal::JSON_VALUE_MASK; } -really_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { +simdjson_really_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { return uint32_t(doc->tape[json_index]); } -really_inline uint32_t internal::tape_ref::scope_count() const noexcept { +simdjson_really_inline uint32_t internal::tape_ref::scope_count() const noexcept { return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK); } template -really_inline T tape_ref::next_tape_value() const noexcept { +simdjson_really_inline T tape_ref::next_tape_value() const noexcept { static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit"); // Though the following is tempting... // return *reinterpret_cast(&doc->tape[json_index + 1]); @@ -7915,14 +7909,14 @@ really_inline T tape_ref::next_tape_value() const noexcept { return x; } -really_inline uint32_t internal::tape_ref::get_string_length() const noexcept { +simdjson_really_inline uint32_t internal::tape_ref::get_string_length() const noexcept { size_t string_buf_index = size_t(tape_value()); uint32_t len; memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); return len; } -really_inline const char * internal::tape_ref::get_c_str() const noexcept { +simdjson_really_inline const char * internal::tape_ref::get_c_str() const noexcept { size_t string_buf_index = size_t(tape_value()); return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); } @@ -7938,9 +7932,9 @@ inline std::string_view internal::tape_ref::get_string_view() const noexcept { } // namespace simdjson #endif // SIMDJSON_INLINE_TAPE_REF_H -/* end file include/simdjson/inline/tape_ref.h */ +/* end file include/simdjson/internal/tape_ref-inl.h */ SIMDJSON_POP_DISABLE_WARNINGS #endif // SIMDJSON_H -/* end file include/simdjson/inline/tape_ref.h */ +/* end file include/simdjson/internal/tape_ref-inl.h */