From af8b52e7e81f646d392470c7f42aedea552cd764 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 16 Jul 2020 12:20:25 -0700 Subject: [PATCH 1/7] Target region for entire compilation of an implementation --- include/simdjson/portability.h | 21 ++++++++------------- src/arm64/begin_implementation.h | 5 +++++ src/arm64/bitmanipulation.h | 1 - src/arm64/bitmask.h | 4 +--- src/arm64/dom_parser_implementation.cpp | 14 ++++++++------ src/arm64/end_implementation.h | 1 + src/arm64/implementation.cpp | 11 ++++------- src/arm64/numberparsing.h | 1 - src/arm64/simd.h | 1 - src/arm64/stringparsing.h | 1 - src/fallback/begin_implementation.h | 4 ++++ src/fallback/dom_parser_implementation.cpp | 14 ++++++++------ src/fallback/end_implementation.h | 1 + src/fallback/implementation.cpp | 11 ++++------- src/fallback/implementation.h | 1 - src/haswell/begin_implementation.h | 6 ++++++ src/haswell/bitmanipulation.h | 8 ++------ src/haswell/bitmask.h | 9 ++------- src/haswell/dom_parser_implementation.cpp | 18 ++++++++---------- src/haswell/dom_parser_implementation.h | 4 ++-- src/haswell/end_implementation.h | 2 ++ src/haswell/implementation.cpp | 12 +++++------- src/haswell/implementation.h | 1 + src/haswell/intrinsics.h | 11 +++++++---- src/haswell/numberparsing.h | 9 +++------ src/haswell/simd.h | 7 ++----- src/haswell/stringparsing.h | 7 ++----- src/westmere/begin_implementation.h | 6 ++++++ src/westmere/bitmanipulation.h | 8 ++------ src/westmere/bitmask.h | 8 ++------ src/westmere/dom_parser_implementation.cpp | 19 ++++++++----------- src/westmere/end_implementation.h | 2 ++ src/westmere/implementation.cpp | 11 ++++------- src/westmere/implementation.h | 1 + src/westmere/numberparsing.h | 10 +++------- src/westmere/simd.h | 10 ++-------- src/westmere/stringparsing.h | 7 ++----- 37 files changed, 118 insertions(+), 149 deletions(-) create mode 100644 src/arm64/begin_implementation.h create mode 100644 src/arm64/end_implementation.h create mode 100644 src/fallback/begin_implementation.h create mode 100644 src/fallback/end_implementation.h create mode 100644 src/haswell/begin_implementation.h create mode 100644 src/haswell/end_implementation.h create mode 100644 src/westmere/begin_implementation.h create mode 100644 src/westmere/end_implementation.h diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h index e91a2423..d6a1b8d7 100644 --- a/include/simdjson/portability.h +++ b/include/simdjson/portability.h @@ -103,32 +103,27 @@ use a 64-bit target such as x64 or 64-bit ARM.") #ifdef __clang__ // clang does not have GCC push pop // warning: clang attribute push can't be used within a namespace in clang up -// til 8.0 so TARGET_REGION and UNTARGET_REGION must be *outside* of a +// til 8.0 so SIMDJSON_TARGET_REGION and SIMDJSON_UNTARGET_REGION must be *outside* of a // namespace. -#define TARGET_REGION(T) \ +#define SIMDJSON_TARGET_REGION(T) \ _Pragma(STRINGIFY( \ clang attribute push(__attribute__((target(T))), apply_to = function))) -#define UNTARGET_REGION _Pragma("clang attribute pop") +#define SIMDJSON_UNTARGET_REGION _Pragma("clang attribute pop") #elif defined(__GNUC__) // GCC is easier -#define TARGET_REGION(T) \ +#define SIMDJSON_TARGET_REGION(T) \ _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) -#define UNTARGET_REGION _Pragma("GCC pop_options") +#define SIMDJSON_UNTARGET_REGION _Pragma("GCC pop_options") #endif // clang then gcc #endif // x86 // Default target region macros don't do anything. -#ifndef TARGET_REGION -#define TARGET_REGION(T) -#define UNTARGET_REGION +#ifndef SIMDJSON_TARGET_REGION +#define SIMDJSON_TARGET_REGION(T) +#define SIMDJSON_UNTARGET_REGION #endif -// under GCC and CLANG, we use these two macros -#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul,lzcnt") -#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") -#define TARGET_ARM64 - // Is threading enabled? #if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT) #ifndef SIMDJSON_THREADS_ENABLED diff --git a/src/arm64/begin_implementation.h b/src/arm64/begin_implementation.h new file mode 100644 index 00000000..aba32298 --- /dev/null +++ b/src/arm64/begin_implementation.h @@ -0,0 +1,5 @@ +#include "simdjson.h" +#include "arm64/implementation.h" +#include "arm64/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION + +#define SIMDJSON_IMPLEMENTATION arm64 diff --git a/src/arm64/bitmanipulation.h b/src/arm64/bitmanipulation.h index fe002043..7cab8016 100644 --- a/src/arm64/bitmanipulation.h +++ b/src/arm64/bitmanipulation.h @@ -2,7 +2,6 @@ #define SIMDJSON_ARM64_BITMANIPULATION_H #include "simdjson.h" -#include "arm64/intrinsics.h" namespace simdjson { namespace arm64 { diff --git a/src/arm64/bitmask.h b/src/arm64/bitmask.h index 3c8ceed9..fbaf6aa9 100644 --- a/src/arm64/bitmask.h +++ b/src/arm64/bitmask.h @@ -3,8 +3,6 @@ #include "simdjson.h" -#include "arm64/intrinsics.h" - namespace simdjson { namespace arm64 { @@ -38,6 +36,6 @@ really_inline uint64_t prefix_xor(uint64_t bitmask) { } // namespace arm64 } // namespace simdjson -UNTARGET_REGION +SIMDJSON_UNTARGET_REGION #endif diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 5ac421d4..e7debac2 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -1,5 +1,4 @@ -#include "simdjson.h" -#include "arm64/implementation.h" +#include "arm64/begin_implementation.h" #include "arm64/dom_parser_implementation.h" // @@ -10,7 +9,7 @@ #include "arm64/bitmanipulation.h" namespace simdjson { -namespace arm64 { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -132,7 +131,8 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return simdjson::arm64::stage1::generic_validate_utf8(buf,len); } -} // namespace arm64 + +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson // @@ -143,7 +143,7 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons #include "arm64/numberparsing.h" namespace simdjson { -namespace arm64 { +namespace SIMDJSON_IMPLEMENTATION { #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" @@ -156,5 +156,7 @@ WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, siz return stage2(_doc); } -} // namespace arm64 +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson + +#include "arm64/end_implementation.h" \ No newline at end of file diff --git a/src/arm64/end_implementation.h b/src/arm64/end_implementation.h new file mode 100644 index 00000000..69c9f4b8 --- /dev/null +++ b/src/arm64/end_implementation.h @@ -0,0 +1 @@ +#undef SIMDJSON_IMPLEMENTATION diff --git a/src/arm64/implementation.cpp b/src/arm64/implementation.cpp index 7276aa29..a4b8958d 100644 --- a/src/arm64/implementation.cpp +++ b/src/arm64/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "arm64/implementation.h" +#include "arm64/begin_implementation.h" #include "arm64/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace arm64 { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,7 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace arm64 +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "arm64/end_implementation.h" \ No newline at end of file diff --git a/src/arm64/numberparsing.h b/src/arm64/numberparsing.h index 7355c687..0854b469 100644 --- a/src/arm64/numberparsing.h +++ b/src/arm64/numberparsing.h @@ -3,7 +3,6 @@ #include "simdjson.h" #include "jsoncharutils.h" -#include "arm64/intrinsics.h" #include "arm64/bitmanipulation.h" #include #include diff --git a/src/arm64/simd.h b/src/arm64/simd.h index c00ae99e..e3665427 100644 --- a/src/arm64/simd.h +++ b/src/arm64/simd.h @@ -4,7 +4,6 @@ #include "simdjson.h" #include "simdprune_tables.h" #include "arm64/bitmanipulation.h" -#include "arm64/intrinsics.h" #include diff --git a/src/arm64/stringparsing.h b/src/arm64/stringparsing.h index 0673e04c..98decef2 100644 --- a/src/arm64/stringparsing.h +++ b/src/arm64/stringparsing.h @@ -4,7 +4,6 @@ #include "simdjson.h" #include "jsoncharutils.h" #include "arm64/simd.h" -#include "arm64/intrinsics.h" #include "arm64/bitmanipulation.h" namespace simdjson { diff --git a/src/fallback/begin_implementation.h b/src/fallback/begin_implementation.h new file mode 100644 index 00000000..f0ad486e --- /dev/null +++ b/src/fallback/begin_implementation.h @@ -0,0 +1,4 @@ +#include "simdjson.h" +#include "fallback/implementation.h" + +#define SIMDJSON_IMPLEMENTATION fallback diff --git a/src/fallback/dom_parser_implementation.cpp b/src/fallback/dom_parser_implementation.cpp index a474b68f..44e98b4e 100644 --- a/src/fallback/dom_parser_implementation.cpp +++ b/src/fallback/dom_parser_implementation.cpp @@ -1,12 +1,12 @@ -#include "simdjson.h" -#include "fallback/implementation.h" +#include "fallback/begin_implementation.h" #include "fallback/dom_parser_implementation.h" // // Stage 1 // namespace simdjson { -namespace fallback { +namespace SIMDJSON_IMPLEMENTATION { + namespace stage1 { #include "generic/stage1/find_next_document_index.h" @@ -308,7 +308,7 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons return true; } -} // namespace fallback +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson // @@ -318,7 +318,7 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons #include "fallback/numberparsing.h" namespace simdjson { -namespace fallback { +namespace SIMDJSON_IMPLEMENTATION { #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" @@ -331,5 +331,7 @@ WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, siz return stage2(_doc); } -} // namespace fallback +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson + +#include "fallback/end_implementation.h" \ No newline at end of file diff --git a/src/fallback/end_implementation.h b/src/fallback/end_implementation.h new file mode 100644 index 00000000..69c9f4b8 --- /dev/null +++ b/src/fallback/end_implementation.h @@ -0,0 +1 @@ +#undef SIMDJSON_IMPLEMENTATION diff --git a/src/fallback/implementation.cpp b/src/fallback/implementation.cpp index 3981f222..305e040c 100644 --- a/src/fallback/implementation.cpp +++ b/src/fallback/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "fallback/implementation.h" +#include "fallback/begin_implementation.h" #include "fallback/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace fallback { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,7 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace fallback +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "fallback/end_implementation.h" \ No newline at end of file diff --git a/src/fallback/implementation.h b/src/fallback/implementation.h index 5d92d385..5b99218c 100644 --- a/src/fallback/implementation.h +++ b/src/fallback/implementation.h @@ -26,7 +26,6 @@ public: }; } // namespace fallback - } // namespace simdjson #endif // SIMDJSON_FALLBACK_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/haswell/begin_implementation.h b/src/haswell/begin_implementation.h new file mode 100644 index 00000000..bfa87e29 --- /dev/null +++ b/src/haswell/begin_implementation.h @@ -0,0 +1,6 @@ +#include "simdjson.h" +#include "haswell/implementation.h" +#include "haswell/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION + +#define SIMDJSON_IMPLEMENTATION haswell +SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt") diff --git a/src/haswell/bitmanipulation.h b/src/haswell/bitmanipulation.h index f6b8fc72..24042649 100644 --- a/src/haswell/bitmanipulation.h +++ b/src/haswell/bitmanipulation.h @@ -3,11 +3,8 @@ #include "simdjson.h" -#include "haswell/intrinsics.h" - -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { // We sometimes call trailing_zero on inputs that are zero, // but the algorithms do not end up using the returned value. @@ -73,8 +70,7 @@ really_inline bool mul_overflow(uint64_t value1, uint64_t value2, #endif } -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_BITMANIPULATION_H diff --git a/src/haswell/bitmask.h b/src/haswell/bitmask.h index c9ed2e0e..9661f7ef 100644 --- a/src/haswell/bitmask.h +++ b/src/haswell/bitmask.h @@ -3,11 +3,8 @@ #include "simdjson.h" -#include "haswell/intrinsics.h" - -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { // // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. @@ -22,9 +19,7 @@ really_inline uint64_t prefix_xor(const uint64_t bitmask) { return _mm_cvtsi128_si64(result); } -} // namespace haswell - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_BITMASK_H diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index 2a1f1891..b104754e 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -1,5 +1,4 @@ -#include "simdjson.h" -#include "haswell/implementation.h" +#include "haswell/begin_implementation.h" #include "haswell/dom_parser_implementation.h" // @@ -9,9 +8,8 @@ #include "haswell/simd.h" #include "haswell/bitmanipulation.h" -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -97,9 +95,9 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return simdjson::haswell::stage1::generic_validate_utf8(buf,len); } -} // namespace haswell + +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION // // Stage 2 @@ -107,9 +105,8 @@ UNTARGET_REGION #include "haswell/stringparsing.h" #include "haswell/numberparsing.h" -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" @@ -122,6 +119,7 @@ WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, siz return stage2(_doc); } -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION + +#include "haswell/end_implementation.h" diff --git a/src/haswell/dom_parser_implementation.h b/src/haswell/dom_parser_implementation.h index fdf7d2b0..180be00a 100644 --- a/src/haswell/dom_parser_implementation.h +++ b/src/haswell/dom_parser_implementation.h @@ -5,11 +5,11 @@ #include "isadetection.h" namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { #include "generic/dom_parser_implementation.h" -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson #endif // SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/haswell/end_implementation.h b/src/haswell/end_implementation.h new file mode 100644 index 00000000..cf78b56c --- /dev/null +++ b/src/haswell/end_implementation.h @@ -0,0 +1,2 @@ +#undef SIMDJSON_IMPLEMENTATION +SIMDJSON_UNTARGET_REGION diff --git a/src/haswell/implementation.cpp b/src/haswell/implementation.cpp index 53daa182..09c38aba 100644 --- a/src/haswell/implementation.cpp +++ b/src/haswell/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "haswell/implementation.h" +#include "haswell/begin_implementation.h" #include "haswell/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,8 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "haswell/end_implementation.h" + diff --git a/src/haswell/implementation.h b/src/haswell/implementation.h index 36e5f389..efd20e30 100644 --- a/src/haswell/implementation.h +++ b/src/haswell/implementation.h @@ -4,6 +4,7 @@ #include "simdjson.h" #include "isadetection.h" +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_REGION namespace simdjson { namespace haswell { diff --git a/src/haswell/intrinsics.h b/src/haswell/intrinsics.h index 5abc48c7..69df3d36 100644 --- a/src/haswell/intrinsics.h +++ b/src/haswell/intrinsics.h @@ -41,12 +41,15 @@ // has it as a macro. #ifndef _blsr_u64 // we roll our own -TARGET_HASWELL -static really_inline uint64_t simdjson_blsr_u64(uint64_t n) { +#include "haswell/begin_implementation.h" +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +static really_inline uint64_t _blsr_u64(uint64_t n) { return (n - 1) & n; } -UNTARGET_REGION -#define _blsr_u64(a) (simdjson_blsr_u64((a))) +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson +#include "haswell/end_implementation.h" #endif // _blsr_u64 #endif diff --git a/src/haswell/numberparsing.h b/src/haswell/numberparsing.h index 6f7e242e..472b62d0 100644 --- a/src/haswell/numberparsing.h +++ b/src/haswell/numberparsing.h @@ -4,7 +4,6 @@ #include "simdjson.h" #include "jsoncharutils.h" -#include "haswell/intrinsics.h" #include "haswell/bitmanipulation.h" #include #include @@ -16,9 +15,9 @@ void found_unsigned_integer(uint64_t result, const uint8_t *buf); void found_float(double result, const uint8_t *buf); #endif -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { + static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); @@ -41,9 +40,7 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) #include "generic/stage2/numberparsing.h" -} // namespace haswell - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_NUMBERPARSING_H diff --git a/src/haswell/simd.h b/src/haswell/simd.h index c95f01df..f2410842 100644 --- a/src/haswell/simd.h +++ b/src/haswell/simd.h @@ -4,11 +4,9 @@ #include "simdjson.h" #include "simdprune_tables.h" #include "haswell/bitmanipulation.h" -#include "haswell/intrinsics.h" -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { namespace simd { // Forward-declared so they can be used by splat and friends. @@ -352,8 +350,7 @@ namespace simd { } // namespace simd -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_SIMD_H diff --git a/src/haswell/stringparsing.h b/src/haswell/stringparsing.h index a8d7728b..a7b52495 100644 --- a/src/haswell/stringparsing.h +++ b/src/haswell/stringparsing.h @@ -4,12 +4,10 @@ #include "simdjson.h" #include "jsoncharutils.h" #include "haswell/simd.h" -#include "haswell/intrinsics.h" #include "haswell/bitmanipulation.h" -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -43,8 +41,7 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 #include "generic/stage2/stringparsing.h" -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_STRINGPARSING_H diff --git a/src/westmere/begin_implementation.h b/src/westmere/begin_implementation.h new file mode 100644 index 00000000..8c7bfd16 --- /dev/null +++ b/src/westmere/begin_implementation.h @@ -0,0 +1,6 @@ +#include "simdjson.h" +#include "westmere/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION +#include "westmere/implementation.h" + +#define SIMDJSON_IMPLEMENTATION westmere +SIMDJSON_TARGET_REGION("sse4.2,pclmul") diff --git a/src/westmere/bitmanipulation.h b/src/westmere/bitmanipulation.h index 29a8623a..3cf20323 100644 --- a/src/westmere/bitmanipulation.h +++ b/src/westmere/bitmanipulation.h @@ -2,11 +2,9 @@ #define SIMDJSON_WESTMERE_BITMANIPULATION_H #include "simdjson.h" -#include "westmere/intrinsics.h" -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { // We sometimes call trailing_zero on inputs that are zero, // but the algorithms do not end up using the returned value. @@ -81,9 +79,7 @@ really_inline bool mul_overflow(uint64_t value1, uint64_t value2, #endif } -} // namespace westmere - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_BITMANIPULATION_H diff --git a/src/westmere/bitmask.h b/src/westmere/bitmask.h index 32bab6da..d1da0eb1 100644 --- a/src/westmere/bitmask.h +++ b/src/westmere/bitmask.h @@ -2,11 +2,9 @@ #define SIMDJSON_WESTMERE_BITMASK_H #include "simdjson.h" -#include "westmere/intrinsics.h" -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { // // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. @@ -21,9 +19,7 @@ really_inline uint64_t prefix_xor(const uint64_t bitmask) { return _mm_cvtsi128_si64(result); } -} // namespace westmere - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_BITMASK_H diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index c665e76c..ff156970 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -1,6 +1,4 @@ -#include "simdjson.h" -#include "westmere/implementation.h" -#include "westmere/dom_parser_implementation.h" +#include "westmere/begin_implementation.h" // // Stage 1 @@ -10,9 +8,8 @@ #include "westmere/bitmanipulation.h" #include "westmere/implementation.h" -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -103,9 +100,9 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return simdjson::westmere::stage1::generic_validate_utf8(buf,len); } -} // namespace westmere + +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION // // Stage 2 @@ -113,9 +110,8 @@ UNTARGET_REGION #include "westmere/stringparsing.h" #include "westmere/numberparsing.h" -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" @@ -128,6 +124,7 @@ WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, siz return stage2(_doc); } -} // namespace westmere +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION + +#include "westmere/end_implementation.h" diff --git a/src/westmere/end_implementation.h b/src/westmere/end_implementation.h new file mode 100644 index 00000000..cf78b56c --- /dev/null +++ b/src/westmere/end_implementation.h @@ -0,0 +1,2 @@ +#undef SIMDJSON_IMPLEMENTATION +SIMDJSON_UNTARGET_REGION diff --git a/src/westmere/implementation.cpp b/src/westmere/implementation.cpp index 56792a8c..e166210b 100644 --- a/src/westmere/implementation.cpp +++ b/src/westmere/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "westmere/implementation.h" +#include "westmere/begin_implementation.h" #include "westmere/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,7 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace westmere +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "westmere/end_implementation.h" \ No newline at end of file diff --git a/src/westmere/implementation.h b/src/westmere/implementation.h index 69dceb20..3fe8e6ac 100644 --- a/src/westmere/implementation.h +++ b/src/westmere/implementation.h @@ -5,6 +5,7 @@ #include "simdjson/implementation.h" #include "isadetection.h" +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_REGION namespace simdjson { namespace westmere { diff --git a/src/westmere/numberparsing.h b/src/westmere/numberparsing.h index fa856e00..f10fc77c 100644 --- a/src/westmere/numberparsing.h +++ b/src/westmere/numberparsing.h @@ -3,7 +3,6 @@ #include "simdjson.h" #include "jsoncharutils.h" -#include "westmere/intrinsics.h" #include "westmere/bitmanipulation.h" #include #include @@ -16,10 +15,9 @@ void found_unsigned_integer(uint64_t result, const uint8_t *buf); void found_float(double result, const uint8_t *buf); #endif - -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { + static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); @@ -42,9 +40,7 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) #include "generic/stage2/numberparsing.h" -} // namespace westmere - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_NUMBERPARSING_H diff --git a/src/westmere/simd.h b/src/westmere/simd.h index efbe508b..597e2fce 100644 --- a/src/westmere/simd.h +++ b/src/westmere/simd.h @@ -4,13 +4,9 @@ #include "simdjson.h" #include "simdprune_tables.h" #include "westmere/bitmanipulation.h" -#include "westmere/intrinsics.h" - - -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { namespace simd { template @@ -333,9 +329,7 @@ namespace simd { }; // struct simd8x64 } // namespace simd - -} // namespace westmere +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_SIMD_INPUT_H diff --git a/src/westmere/stringparsing.h b/src/westmere/stringparsing.h index e2b0a4ac..09f34318 100644 --- a/src/westmere/stringparsing.h +++ b/src/westmere/stringparsing.h @@ -4,12 +4,10 @@ #include "simdjson.h" #include "jsoncharutils.h" #include "westmere/simd.h" -#include "westmere/intrinsics.h" #include "westmere/bitmanipulation.h" -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -45,8 +43,7 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 #include "generic/stage2/stringparsing.h" -} // namespace westmere +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_STRINGPARSING_H From 464f4813e39449811f02f1672cc44d927649d470 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 16 Jul 2020 12:56:02 -0700 Subject: [PATCH 2/7] Define namespaces inside generic files --- src/arm64/dom_parser_implementation.cpp | 66 +++---- src/arm64/dom_parser_implementation.h | 9 - src/arm64/numberparsing.h | 6 +- src/arm64/stringparsing.h | 4 +- src/fallback/dom_parser_implementation.cpp | 13 +- src/fallback/dom_parser_implementation.h | 9 - src/fallback/numberparsing.h | 12 +- src/fallback/stringparsing.h | 4 +- src/generic/dom_parser_implementation.h | 15 ++ src/generic/stage1/allocate.h | 4 + src/generic/stage1/buf_block_reader.h | 6 + src/generic/stage1/find_next_document_index.h | 6 + src/generic/stage1/json_minifier.h | 4 + src/generic/stage1/json_scanner.h | 4 + src/generic/stage1/json_string_scanner.h | 6 +- src/generic/stage1/json_structural_indexer.h | 4 + .../stage1/utf8_fastvalidate_algorithm.h | 6 + src/generic/stage1/utf8_lookup2_algorithm.h | 6 + src/generic/stage1/utf8_lookup3_algorithm.h | 6 + src/generic/stage1/utf8_lookup4_algorithm.h | 10 +- src/generic/stage1/utf8_lookup_algorithm.h | 6 + src/generic/stage1/utf8_range_algorithm.h | 6 + src/generic/stage1/utf8_validator.h | 6 +- src/generic/stage1/utf8_zwegner_algorithm.h | 5 + src/generic/stage2/allocate.h | 4 + src/generic/stage2/atomparsing.h | 4 + src/generic/stage2/logger.h | 5 +- src/generic/stage2/numberparsing.h | 4 + .../stage2/streaming_structural_parser.h | 168 ------------------ src/generic/stage2/stringparsing.h | 6 +- src/generic/stage2/structural_iterator.h | 4 + src/generic/stage2/structural_parser.h | 12 +- src/generic/stage2/tape_writer.h | 8 + src/haswell/dom_parser_implementation.cpp | 61 ++++--- src/haswell/dom_parser_implementation.h | 9 - src/haswell/numberparsing.h | 6 +- src/haswell/stringparsing.h | 4 +- src/westmere/dom_parser_implementation.cpp | 61 ++++--- src/westmere/dom_parser_implementation.h | 9 - src/westmere/numberparsing.h | 6 +- src/westmere/stringparsing.h | 4 +- 41 files changed, 266 insertions(+), 332 deletions(-) delete mode 100755 src/generic/stage2/streaming_structural_parser.h diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index e7debac2..448ffc5e 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -101,39 +101,16 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c return is_third_byte ^ is_fourth_byte; } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #include "generic/stage1/buf_block_reader.h" #include "generic/stage1/json_string_scanner.h" #include "generic/stage1/json_scanner.h" - -namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no - // benefit and therefore makes things worse. - // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} -} - #include "generic/stage1/json_minifier.h" -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - #include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { - this->buf = _buf; - this->len = _len; - return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); -} -#include "generic/stage1/utf8_validator.h" -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return simdjson::arm64::stage1::generic_validate_utf8(buf,len); -} - -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson // // Stage 2 @@ -141,15 +118,44 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons #include "arm64/stringparsing.h" #include "arm64/numberparsing.h" - -namespace simdjson { -namespace SIMDJSON_IMPLEMENTATION { - #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" #include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" +// +// Implementation-specific overrides +// +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + +namespace stage1 { + +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no + // benefit and therefore makes things worse. + // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} + +} // namespace stage1 + +WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { + return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); +} + +WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { + this->buf = _buf; + this->len = _len; + return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); +} + +#include "generic/stage1/utf8_validator.h" + +WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return simdjson::arm64::stage1::generic_validate_utf8(buf,len); +} + WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); if (err) { return err; } diff --git a/src/arm64/dom_parser_implementation.h b/src/arm64/dom_parser_implementation.h index 29174f80..888b3422 100644 --- a/src/arm64/dom_parser_implementation.h +++ b/src/arm64/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace arm64 { - #include "generic/dom_parser_implementation.h" -} // namespace arm64 -} // namespace simdjson - #endif // SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/arm64/numberparsing.h b/src/arm64/numberparsing.h index 0854b469..758a350f 100644 --- a/src/arm64/numberparsing.h +++ b/src/arm64/numberparsing.h @@ -28,11 +28,11 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); } +} // namespace arm64 +} // namespace simdjson + #define SWAR_NUMBER_PARSING #include "generic/stage2/numberparsing.h" -} // namespace arm64 -} // namespace simdjson - #endif // SIMDJSON_ARM64_NUMBERPARSING_H diff --git a/src/arm64/stringparsing.h b/src/arm64/stringparsing.h index 98decef2..39a1af12 100644 --- a/src/arm64/stringparsing.h +++ b/src/arm64/stringparsing.h @@ -44,9 +44,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 }; } -#include "generic/stage2/stringparsing.h" - } // namespace arm64 } // namespace simdjson +#include "generic/stage2/stringparsing.h" + #endif // SIMDJSON_ARM64_STRINGPARSING_H diff --git a/src/fallback/dom_parser_implementation.cpp b/src/fallback/dom_parser_implementation.cpp index 44e98b4e..444ef69e 100644 --- a/src/fallback/dom_parser_implementation.cpp +++ b/src/fallback/dom_parser_implementation.cpp @@ -4,13 +4,12 @@ // // Stage 1 // +#include "generic/stage1/find_next_document_index.h" + namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { - namespace stage1 { -#include "generic/stage1/find_next_document_index.h" - class structural_scanner { public: @@ -180,7 +179,6 @@ private: } // namespace stage1 - WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool partial) noexcept { this->buf = _buf; this->len = _len; @@ -316,15 +314,14 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons // #include "fallback/stringparsing.h" #include "fallback/numberparsing.h" - -namespace simdjson { -namespace SIMDJSON_IMPLEMENTATION { - #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" #include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); if (err) { return err; } diff --git a/src/fallback/dom_parser_implementation.h b/src/fallback/dom_parser_implementation.h index b648c36c..d7c83e07 100644 --- a/src/fallback/dom_parser_implementation.h +++ b/src/fallback/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace fallback { - #include "generic/dom_parser_implementation.h" -} // namespace fallback -} // namespace simdjson - #endif // SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/fallback/numberparsing.h b/src/fallback/numberparsing.h index 75760655..36586a21 100644 --- a/src/fallback/numberparsing.h +++ b/src/fallback/numberparsing.h @@ -15,7 +15,7 @@ void found_float(double result, const uint8_t *buf); #endif namespace simdjson { -namespace fallback { +namespace SIMDJSON_IMPLEMENTATION { static really_inline uint32_t parse_eight_digits_unrolled(const char *chars) { uint32_t result = 0; for (int i=0;i<8;i++) { @@ -27,12 +27,10 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) return parse_eight_digits_unrolled((const char *)chars); } -#define SWAR_NUMBER_PARSING - -#include "generic/stage2/numberparsing.h" - -} // namespace fallback - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson +#define SWAR_NUMBER_PARSING +#include "generic/stage2/numberparsing.h" + #endif // SIMDJSON_FALLBACK_NUMBERPARSING_H diff --git a/src/fallback/stringparsing.h b/src/fallback/stringparsing.h index 599dc8ec..0501ae5a 100644 --- a/src/fallback/stringparsing.h +++ b/src/fallback/stringparsing.h @@ -27,9 +27,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 return { src[0] }; } -#include "generic/stage2/stringparsing.h" - } // namespace fallback } // namespace simdjson +#include "generic/stage2/stringparsing.h" + #endif // SIMDJSON_FALLBACK_STRINGPARSING_H diff --git a/src/generic/dom_parser_implementation.h b/src/generic/dom_parser_implementation.h index ca45b98e..ee59694b 100644 --- a/src/generic/dom_parser_implementation.h +++ b/src/generic/dom_parser_implementation.h @@ -1,3 +1,9 @@ +#include "simdjson.h" +#include "isadetection.h" + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // expectation: sizeof(scope_descriptor) = 64/8. struct scope_descriptor { uint32_t tape_index; // where, on the tape, does the scope ([,{) begins @@ -38,9 +44,15 @@ public: WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; }; +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #include "generic/stage1/allocate.h" #include "generic/stage2/allocate.h" +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + really_inline dom_parser_implementation::dom_parser_implementation() {} // Leaving these here so they can be inlined if so desired @@ -57,3 +69,6 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth _max_depth = max_depth; return SUCCESS; } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/allocate.h b/src/generic/stage1/allocate.h index ab6fdb8a..e9ecb07e 100644 --- a/src/generic/stage1/allocate.h +++ b/src/generic/stage1/allocate.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { namespace allocate { @@ -15,3 +17,5 @@ really_inline error_code set_capacity(internal::dom_parser_implementation &parse } // namespace allocate } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/buf_block_reader.h b/src/generic/stage1/buf_block_reader.h index fb5c07e7..6eb28cc7 100644 --- a/src/generic/stage1/buf_block_reader.h +++ b/src/generic/stage1/buf_block_reader.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // Walks through a buffer in block-sized increments, loading the last part with spaces template struct buf_block_reader { @@ -81,3 +84,6 @@ template really_inline void buf_block_reader::advance() { idx += STEP_SIZE; } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/find_next_document_index.h b/src/generic/stage1/find_next_document_index.h index 41d6bdb1..7e42d7f3 100644 --- a/src/generic/stage1/find_next_document_index.h +++ b/src/generic/stage1/find_next_document_index.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + /** * This algorithm is used to quickly identify the last structural position that * makes up a complete document. @@ -84,3 +87,6 @@ really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left return len; } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_minifier.h b/src/generic/stage1/json_minifier.h index 4055d90a..46bfd4e5 100644 --- a/src/generic/stage1/json_minifier.h +++ b/src/generic/stage1/json_minifier.h @@ -3,6 +3,8 @@ // We assume the file in which it is included already includes // "simdjson/stage1.h" (this simplifies amalgation) +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { class json_minifier { @@ -75,3 +77,5 @@ error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, s } } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_scanner.h b/src/generic/stage1/json_scanner.h index 0834c4bd..8544a8cf 100644 --- a/src/generic/stage1/json_scanner.h +++ b/src/generic/stage1/json_scanner.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { /** @@ -102,3 +104,5 @@ really_inline error_code json_scanner::finish(bool streaming) { } } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_string_scanner.h b/src/generic/stage1/json_string_scanner.h index f37855c1..24d24d8c 100644 --- a/src/generic/stage1/json_string_scanner.h +++ b/src/generic/stage1/json_string_scanner.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { struct json_string_block { @@ -136,4 +138,6 @@ really_inline error_code json_string_scanner::finish(bool streaming) { return SUCCESS; } -} // namespace stage1 \ No newline at end of file +} // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index cf6ec261..a2663fe8 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -3,6 +3,8 @@ // We assume the file in which it is included already includes // "simdjson/stage1.h" (this simplifies amalgation) +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { class bit_indexer { @@ -196,3 +198,5 @@ really_inline error_code json_structural_indexer::finish(dom_parser_implementati } } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_fastvalidate_algorithm.h b/src/generic/stage1/utf8_fastvalidate_algorithm.h index ea6a88ce..fb9b95dc 100644 --- a/src/generic/stage1/utf8_fastvalidate_algorithm.h +++ b/src/generic/stage1/utf8_fastvalidate_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + /* * legal utf-8 byte sequence * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 @@ -176,3 +179,6 @@ struct utf8_checker { return this->has_error.any_bits_set_anywhere() ? simdjson::UTF8_ERROR : simdjson::SUCCESS; } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup2_algorithm.h b/src/generic/stage1/utf8_lookup2_algorithm.h index a5d7ca9a..6badfa07 100644 --- a/src/generic/stage1/utf8_lookup2_algorithm.h +++ b/src/generic/stage1/utf8_lookup2_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // // Detect Unicode errors. // @@ -216,3 +219,6 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup3_algorithm.h b/src/generic/stage1/utf8_lookup3_algorithm.h index e49cffcc..c8e439c2 100644 --- a/src/generic/stage1/utf8_lookup3_algorithm.h +++ b/src/generic/stage1/utf8_lookup3_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // // Detect Unicode errors. // @@ -236,3 +239,6 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup4_algorithm.h b/src/generic/stage1/utf8_lookup4_algorithm.h index 74573510..1bb0fcac 100644 --- a/src/generic/stage1/utf8_lookup4_algorithm.h +++ b/src/generic/stage1/utf8_lookup4_algorithm.h @@ -1,4 +1,5 @@ -using namespace simd; +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace utf8_validation { @@ -173,6 +174,9 @@ using namespace simd; } }; // struct utf8_checker -} +} // namespace utf8_validation -using utf8_validation::utf8_checker; \ No newline at end of file +using utf8_validation::utf8_checker; + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup_algorithm.h b/src/generic/stage1/utf8_lookup_algorithm.h index ad2b97a5..4ad9b1fb 100644 --- a/src/generic/stage1/utf8_lookup_algorithm.h +++ b/src/generic/stage1/utf8_lookup_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // // Detect Unicode errors. // @@ -297,3 +300,6 @@ struct utf8_checker { } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_range_algorithm.h b/src/generic/stage1/utf8_range_algorithm.h index 4caba6dc..ac301bb8 100644 --- a/src/generic/stage1/utf8_range_algorithm.h +++ b/src/generic/stage1/utf8_range_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + /* * legal utf-8 byte sequence * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 @@ -178,3 +181,6 @@ struct utf8_checker { return this->has_error.any() ? simdjson::UTF8_ERROR : simdjson::SUCCESS; } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_validator.h b/src/generic/stage1/utf8_validator.h index cd05cd09..2851d793 100644 --- a/src/generic/stage1/utf8_validator.h +++ b/src/generic/stage1/utf8_validator.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { /** * Validates that the string is actual UTF-8. @@ -23,4 +25,6 @@ bool generic_validate_utf8(const char * input, size_t length) { return generic_validate_utf8((const uint8_t *)input,length); } -} // namespace stage1 \ No newline at end of file +} // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_zwegner_algorithm.h b/src/generic/stage1/utf8_zwegner_algorithm.h index eb9bd298..be81648c 100644 --- a/src/generic/stage1/utf8_zwegner_algorithm.h +++ b/src/generic/stage1/utf8_zwegner_algorithm.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { // // Detect UTF-8 errors. // @@ -358,3 +360,6 @@ struct utf8_checker { return (this->special_case_errors.any_bits_set_anywhere() | this->length_errors) ? simdjson::UTF8_ERROR : simdjson::SUCCESS; } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/allocate.h b/src/generic/stage2/allocate.h index ad35df11..e33a4847 100644 --- a/src/generic/stage2/allocate.h +++ b/src/generic/stage2/allocate.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace allocate { @@ -16,3 +18,5 @@ really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t } // namespace allocate } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/atomparsing.h b/src/generic/stage2/atomparsing.h index 2a23c525..c0d5f78f 100644 --- a/src/generic/stage2/atomparsing.h +++ b/src/generic/stage2/atomparsing.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace atomparsing { @@ -60,3 +62,5 @@ really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { } // namespace atomparsing } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/logger.h b/src/generic/stage2/logger.h index 2f543840..9da6c1ef 100644 --- a/src/generic/stage2/logger.h +++ b/src/generic/stage2/logger.h @@ -1,5 +1,7 @@ // This is for an internal-only stage 2 specific logger. // Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace logger { static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; @@ -61,4 +63,5 @@ namespace logger { } } } // namespace logger - +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/numberparsing.h b/src/generic/stage2/numberparsing.h index 16255baf..14e8d957 100644 --- a/src/generic/stage2/numberparsing.h +++ b/src/generic/stage2/numberparsing.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace numberparsing { @@ -472,3 +474,5 @@ really_inline bool parse_number(const uint8_t *const src, W &writer) { } // namespace numberparsing } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/streaming_structural_parser.h b/src/generic/stage2/streaming_structural_parser.h deleted file mode 100755 index 8e63d028..00000000 --- a/src/generic/stage2/streaming_structural_parser.h +++ /dev/null @@ -1,168 +0,0 @@ -namespace stage2 { - -struct streaming_structural_parser: structural_parser { - really_inline streaming_structural_parser(dom_parser_implementation &_parser) : structural_parser(_parser, _parser.next_structural_index) {} - - // override to add streaming - WARN_UNUSED really_inline error_code start(ret_address_t finish_parser) { - // If there are no structurals left, return EMPTY - if (structurals.at_end(parser.n_structural_indexes)) { - return parser.error = EMPTY; - } - - log_start(); - init(); - - // Capacity ain't no thang for streaming, so we don't check it. - // Advance to the first character as soon as possible - advance_char(); - // Push the root scope (there is always at least one scope) - if (start_document(finish_parser)) { - return parser.error = DEPTH_ERROR; - } - return SUCCESS; - } - - // override to add streaming - WARN_UNUSED really_inline error_code finish() { - if ( structurals.past_end(parser.n_structural_indexes) ) { - log_error("IMPOSSIBLE: past the end of the JSON!"); - return parser.error = TAPE_ERROR; - } - end_document(); - parser.next_structural_index = uint32_t(structurals.next_structural_index()); - if (depth != 0) { - log_error("Unclosed objects or arrays!"); - return parser.error = TAPE_ERROR; - } - if (parser.containing_scope[depth].tape_index != 0) { - log_error("IMPOSSIBLE: root scope tape index did not start at 0!"); - return parser.error = TAPE_ERROR; - } - return SUCCESS; - } -}; - -} // namespace stage2 - -/************ - * The JSON is parsed to a tape, see the accompanying tape.md file - * for documentation. - ***********/ -WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - this->doc = &_doc; - static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); - stage2::streaming_structural_parser parser(*this); - error_code result = parser.start(addresses.finish); - if (result) { return result; } - // - // Read first value - // - switch (parser.structurals.current_char()) { - case '{': - FAIL_IF( parser.start_object(addresses.finish) ); - goto object_begin; - case '[': - FAIL_IF( parser.start_array(addresses.finish) ); - goto array_begin; - case '"': - FAIL_IF( parser.parse_string() ); - goto finish; - case 't': case 'f': case 'n': - FAIL_IF( parser.parse_single_atom() ); - goto finish; - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - FAIL_IF( - parser.structurals.with_space_terminated_copy([&](const uint8_t *copy, size_t idx) { - return parser.parse_number(©[idx], false); - }) - ); - goto finish; - case '-': - FAIL_IF( - parser.structurals.with_space_terminated_copy([&](const uint8_t *copy, size_t idx) { - return parser.parse_number(©[idx], true); - }) - ); - goto finish; - default: - parser.log_error("Document starts with a non-value character"); - goto error; - } - -// -// Object parser parsers -// -object_begin: - switch (parser.advance_char()) { - case '"': { - FAIL_IF( parser.parse_string(true) ); - goto object_key_parser; - } - case '}': - parser.end_object(); - goto scope_end; - default: - parser.log_error("Object does not start with a key"); - goto error; - } - -object_key_parser: - if (parser.advance_char() != ':' ) { parser.log_error("Missing colon after key in object"); goto error; } - parser.increment_count(); - parser.advance_char(); - GOTO( parser.parse_value(addresses, addresses.object_continue) ); - -object_continue: - switch (parser.advance_char()) { - case ',': - if (parser.advance_char() != '"' ) { parser.log_error("Key string missing at beginning of field in object"); goto error; } - FAIL_IF( parser.parse_string(true) ); - goto object_key_parser; - case '}': - parser.end_object(); - goto scope_end; - default: - parser.log_error("No comma between object fields"); - goto error; - } - -scope_end: - CONTINUE( parser.parser.ret_address[parser.depth] ); - -// -// Array parser parsers -// -array_begin: - if (parser.advance_char() == ']') { - parser.end_array(); - goto scope_end; - } - parser.increment_count(); - -main_array_switch: - /* we call update char on all paths in, so we can peek at parser.c on the - * on paths that can accept a close square brace (post-, and at start) */ - GOTO( parser.parse_value(addresses, addresses.array_continue) ); - -array_continue: - switch (parser.advance_char()) { - case ',': - parser.increment_count(); - parser.advance_char(); - goto main_array_switch; - case ']': - parser.end_array(); - goto scope_end; - default: - parser.log_error("Missing comma between array values"); - goto error; - } - -finish: - return parser.finish(); - -error: - return parser.error(); -} diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index 3c8b6419..c777cfe7 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -3,6 +3,8 @@ // We assume the file in which it is include already includes // "stringparsing.h" (this simplifies amalgation) +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace stringparsing { @@ -120,4 +122,6 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst } } // namespace stringparsing -} // namespace stage2 \ No newline at end of file +} // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/structural_iterator.h b/src/generic/stage2/structural_iterator.h index 2682b6d0..c1c7b759 100644 --- a/src/generic/stage2/structural_iterator.h +++ b/src/generic/stage2/structural_iterator.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { class structural_iterator { @@ -44,3 +46,5 @@ public: }; } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/structural_parser.h b/src/generic/stage2/structural_parser.h index d6ca1aca..adaa8913 100644 --- a/src/generic/stage2/structural_parser.h +++ b/src/generic/stage2/structural_parser.h @@ -3,11 +3,14 @@ // We assume the file in which it is include already includes // "simdjson/stage2.h" (this simplifies amalgation) -namespace stage2 { -namespace { // Make everything here private - #include "generic/stage2/tape_writer.h" +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace stage2 { + +namespace { // Make everything here private + #ifdef SIMDJSON_USE_COMPUTED_GOTO #define INIT_ADDRESSES() { &&array_begin, &&array_continue, &&error, &&finish, &&object_begin, &&object_continue } #define GOTO(address) { goto *(address); } @@ -489,3 +492,6 @@ WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) no WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::parse_structurals(*this, _doc); } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/tape_writer.h b/src/generic/stage2/tape_writer.h index 47282d6a..aed7e91f 100644 --- a/src/generic/stage2/tape_writer.h +++ b/src/generic/stage2/tape_writer.h @@ -1,3 +1,7 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace stage2 { + struct tape_writer { /** The next place to write to tape */ uint64_t *next_tape_loc; @@ -93,3 +97,7 @@ really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_typ really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } + +} // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index b104754e..40c4256f 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -66,53 +66,58 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson #include "generic/stage1/buf_block_reader.h" #include "generic/stage1/json_string_scanner.h" #include "generic/stage1/json_scanner.h" - -namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} -} - #include "generic/stage1/json_minifier.h" -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); -} - #include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { - this->buf = _buf; - this->len = _len; - return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); -} -#include "generic/stage1/utf8_validator.h" -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return simdjson::haswell::stage1::generic_validate_utf8(buf,len); -} - -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson // // Stage 2 // #include "haswell/stringparsing.h" #include "haswell/numberparsing.h" - -namespace simdjson { -namespace SIMDJSON_IMPLEMENTATION { - #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" #include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" +// +// Implementation-specific overrides +// +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + +namespace stage1 { + +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} + +} // namespace stage1 + +WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { + return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); +} + +WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { + this->buf = _buf; + this->len = _len; + return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); +} + +#include "generic/stage1/utf8_validator.h" + +WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return simdjson::haswell::stage1::generic_validate_utf8(buf,len); +} + WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); if (err) { return err; } diff --git a/src/haswell/dom_parser_implementation.h b/src/haswell/dom_parser_implementation.h index 180be00a..15d7808a 100644 --- a/src/haswell/dom_parser_implementation.h +++ b/src/haswell/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace SIMDJSON_IMPLEMENTATION { - #include "generic/dom_parser_implementation.h" -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson - #endif // SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/haswell/numberparsing.h b/src/haswell/numberparsing.h index 472b62d0..6bb0b0c2 100644 --- a/src/haswell/numberparsing.h +++ b/src/haswell/numberparsing.h @@ -36,11 +36,11 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) t4); // only captures the sum of the first 8 digits, drop the rest } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #define SWAR_NUMBER_PARSING #include "generic/stage2/numberparsing.h" -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson - #endif // SIMDJSON_HASWELL_NUMBERPARSING_H diff --git a/src/haswell/stringparsing.h b/src/haswell/stringparsing.h index a7b52495..ce33f8e8 100644 --- a/src/haswell/stringparsing.h +++ b/src/haswell/stringparsing.h @@ -39,9 +39,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 }; } -#include "generic/stage2/stringparsing.h" - } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson +#include "generic/stage2/stringparsing.h" + #endif // SIMDJSON_HASWELL_STRINGPARSING_H diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index ff156970..69fb7b9d 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -71,53 +71,58 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson #include "generic/stage1/buf_block_reader.h" #include "generic/stage1/json_string_scanner.h" #include "generic/stage1/json_scanner.h" - -namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} -} - #include "generic/stage1/json_minifier.h" -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - #include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { - this->buf = _buf; - this->len = _len; - return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); -} -#include "generic/stage1/utf8_validator.h" -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return simdjson::westmere::stage1::generic_validate_utf8(buf,len); -} - -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson // // Stage 2 // #include "westmere/stringparsing.h" #include "westmere/numberparsing.h" - -namespace simdjson { -namespace SIMDJSON_IMPLEMENTATION { - #include "generic/stage2/logger.h" #include "generic/stage2/atomparsing.h" #include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" +// +// Implementation-specific overrides +// +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + +namespace stage1 { + +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} + +} // namespace stage1 + +WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { + return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); +} + +WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { + this->buf = _buf; + this->len = _len; + return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); +} + +#include "generic/stage1/utf8_validator.h" + +WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return simdjson::westmere::stage1::generic_validate_utf8(buf,len); +} + WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); if (err) { return err; } diff --git a/src/westmere/dom_parser_implementation.h b/src/westmere/dom_parser_implementation.h index c36d9c8a..56e518b0 100644 --- a/src/westmere/dom_parser_implementation.h +++ b/src/westmere/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_WESTMERE_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_WESTMERE_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace westmere { - #include "generic/dom_parser_implementation.h" -} // namespace westmere -} // namespace simdjson - #endif // SIMDJSON_WESTMERE_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/westmere/numberparsing.h b/src/westmere/numberparsing.h index f10fc77c..ea9c0478 100644 --- a/src/westmere/numberparsing.h +++ b/src/westmere/numberparsing.h @@ -36,11 +36,11 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) t4); // only captures the sum of the first 8 digits, drop the rest } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #define SWAR_NUMBER_PARSING #include "generic/stage2/numberparsing.h" -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson - #endif // SIMDJSON_WESTMERE_NUMBERPARSING_H diff --git a/src/westmere/stringparsing.h b/src/westmere/stringparsing.h index 09f34318..1cc796d5 100644 --- a/src/westmere/stringparsing.h +++ b/src/westmere/stringparsing.h @@ -41,9 +41,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 }; } -#include "generic/stage2/stringparsing.h" - } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson +#include "generic/stage2/stringparsing.h" + #endif // SIMDJSON_WESTMERE_STRINGPARSING_H From 3867ee71ed0d5186020c1a7fd274eb9aae9b4c38 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 16 Jul 2020 13:05:59 -0700 Subject: [PATCH 3/7] Include files where they are used --- src/arm64/dom_parser_implementation.cpp | 13 ++----------- src/fallback/dom_parser_implementation.cpp | 3 --- src/generic/stage1/json_structural_indexer.h | 6 ++++++ src/generic/stage1/utf8_validator.h | 1 + src/generic/stage2/structural_parser.h | 3 +++ src/haswell/dom_parser_implementation.cpp | 11 +---------- src/westmere/dom_parser_implementation.cpp | 11 +---------- 7 files changed, 14 insertions(+), 34 deletions(-) diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 448ffc5e..6d077ad5 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -104,13 +104,9 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#include "generic/stage1/buf_block_reader.h" -#include "generic/stage1/json_string_scanner.h" -#include "generic/stage1/json_scanner.h" -#include "generic/stage1/json_minifier.h" -#include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" +#include "generic/stage1/utf8_validator.h" // // Stage 2 @@ -118,9 +114,6 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c #include "arm64/stringparsing.h" #include "arm64/numberparsing.h" -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" // @@ -150,8 +143,6 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); } -#include "generic/stage1/utf8_validator.h" - WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return simdjson::arm64::stage1::generic_validate_utf8(buf,len); } @@ -165,4 +156,4 @@ WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, siz } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#include "arm64/end_implementation.h" \ No newline at end of file +#include "arm64/end_implementation.h" diff --git a/src/fallback/dom_parser_implementation.cpp b/src/fallback/dom_parser_implementation.cpp index 444ef69e..1df7d868 100644 --- a/src/fallback/dom_parser_implementation.cpp +++ b/src/fallback/dom_parser_implementation.cpp @@ -314,9 +314,6 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons // #include "fallback/stringparsing.h" #include "fallback/numberparsing.h" -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" namespace simdjson { diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index a2663fe8..c0883341 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -3,6 +3,12 @@ // We assume the file in which it is included already includes // "simdjson/stage1.h" (this simplifies amalgation) +#include "generic/stage1/buf_block_reader.h" +#include "generic/stage1/json_string_scanner.h" +#include "generic/stage1/json_scanner.h" +#include "generic/stage1/json_minifier.h" +#include "generic/stage1/find_next_document_index.h" + namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { diff --git a/src/generic/stage1/utf8_validator.h b/src/generic/stage1/utf8_validator.h index 2851d793..5e50166e 100644 --- a/src/generic/stage1/utf8_validator.h +++ b/src/generic/stage1/utf8_validator.h @@ -1,6 +1,7 @@ namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { + /** * Validates that the string is actual UTF-8. */ diff --git a/src/generic/stage2/structural_parser.h b/src/generic/stage2/structural_parser.h index adaa8913..52e2ffa3 100644 --- a/src/generic/stage2/structural_parser.h +++ b/src/generic/stage2/structural_parser.h @@ -4,6 +4,9 @@ // "simdjson/stage2.h" (this simplifies amalgation) #include "generic/stage2/tape_writer.h" +#include "generic/stage2/logger.h" +#include "generic/stage2/atomparsing.h" +#include "generic/stage2/structural_iterator.h" namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index 40c4256f..6e01afb5 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -69,22 +69,15 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#include "generic/stage1/buf_block_reader.h" -#include "generic/stage1/json_string_scanner.h" -#include "generic/stage1/json_scanner.h" -#include "generic/stage1/json_minifier.h" -#include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" +#include "generic/stage1/utf8_validator.h" // // Stage 2 // #include "haswell/stringparsing.h" #include "haswell/numberparsing.h" -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" // @@ -112,8 +105,6 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); } -#include "generic/stage1/utf8_validator.h" - WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return simdjson::haswell::stage1::generic_validate_utf8(buf,len); } diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index 69fb7b9d..4192d2b9 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -74,22 +74,15 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#include "generic/stage1/buf_block_reader.h" -#include "generic/stage1/json_string_scanner.h" -#include "generic/stage1/json_scanner.h" -#include "generic/stage1/json_minifier.h" -#include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" +#include "generic/stage1/utf8_validator.h" // // Stage 2 // #include "westmere/stringparsing.h" #include "westmere/numberparsing.h" -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" // @@ -117,8 +110,6 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); } -#include "generic/stage1/utf8_validator.h" - WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { return simdjson::westmere::stage1::generic_validate_utf8(buf,len); } From 44b7a7145c90782748655dce927de787864bd452 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 16 Jul 2020 13:35:43 -0700 Subject: [PATCH 4/7] Include bitmanip/simd everywhere --- src/arm64/begin_implementation.h | 6 +++++- src/arm64/bitmanipulation.h | 2 -- src/arm64/bitmask.h | 2 -- src/arm64/dom_parser_implementation.cpp | 4 ---- src/arm64/numberparsing.h | 14 -------------- src/arm64/stringparsing.h | 1 - src/fallback/begin_implementation.h | 2 ++ src/fallback/numberparsing.h | 6 ------ src/fallback/stringparsing.h | 1 - src/generic/stage2/numberparsing.h | 4 ++++ src/generic/stage2/stringparsing.h | 4 ++-- src/haswell/begin_implementation.h | 6 ++++++ src/haswell/bitmanipulation.h | 2 -- src/haswell/bitmask.h | 2 -- src/haswell/dom_parser_implementation.cpp | 3 --- src/haswell/numberparsing.h | 14 -------------- src/haswell/simd.h | 2 -- src/haswell/stringparsing.h | 1 - src/westmere/begin_implementation.h | 4 ++++ src/westmere/bitmanipulation.h | 2 -- src/westmere/bitmask.h | 2 -- src/westmere/dom_parser_implementation.cpp | 4 ---- src/westmere/numberparsing.h | 14 -------------- src/westmere/simd.h | 2 -- src/westmere/stringparsing.h | 5 ----- 25 files changed, 23 insertions(+), 86 deletions(-) diff --git a/src/arm64/begin_implementation.h b/src/arm64/begin_implementation.h index aba32298..6cc981e6 100644 --- a/src/arm64/begin_implementation.h +++ b/src/arm64/begin_implementation.h @@ -1,5 +1,9 @@ #include "simdjson.h" #include "arm64/implementation.h" -#include "arm64/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION +#include "arm64/intrinsics.h" #define SIMDJSON_IMPLEMENTATION arm64 + +#include "arm64/bitmanipulation.h" +#include "arm64/bitmask.h" +#include "arm64/simd.h" diff --git a/src/arm64/bitmanipulation.h b/src/arm64/bitmanipulation.h index 7cab8016..bb5a0afe 100644 --- a/src/arm64/bitmanipulation.h +++ b/src/arm64/bitmanipulation.h @@ -1,8 +1,6 @@ #ifndef SIMDJSON_ARM64_BITMANIPULATION_H #define SIMDJSON_ARM64_BITMANIPULATION_H -#include "simdjson.h" - namespace simdjson { namespace arm64 { diff --git a/src/arm64/bitmask.h b/src/arm64/bitmask.h index fbaf6aa9..8e953014 100644 --- a/src/arm64/bitmask.h +++ b/src/arm64/bitmask.h @@ -1,8 +1,6 @@ #ifndef SIMDJSON_ARM64_BITMASK_H #define SIMDJSON_ARM64_BITMASK_H -#include "simdjson.h" - namespace simdjson { namespace arm64 { diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 6d077ad5..0dee412b 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -4,10 +4,6 @@ // // Stage 1 // -#include "arm64/bitmask.h" -#include "arm64/simd.h" -#include "arm64/bitmanipulation.h" - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/arm64/numberparsing.h b/src/arm64/numberparsing.h index 758a350f..d8cc2db0 100644 --- a/src/arm64/numberparsing.h +++ b/src/arm64/numberparsing.h @@ -1,20 +1,6 @@ #ifndef SIMDJSON_ARM64_NUMBERPARSING_H #define SIMDJSON_ARM64_NUMBERPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "arm64/bitmanipulation.h" -#include -#include - - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - namespace simdjson { namespace arm64 { diff --git a/src/arm64/stringparsing.h b/src/arm64/stringparsing.h index 39a1af12..d6e55d6e 100644 --- a/src/arm64/stringparsing.h +++ b/src/arm64/stringparsing.h @@ -2,7 +2,6 @@ #define SIMDJSON_ARM64_STRINGPARSING_H #include "simdjson.h" -#include "jsoncharutils.h" #include "arm64/simd.h" #include "arm64/bitmanipulation.h" diff --git a/src/fallback/begin_implementation.h b/src/fallback/begin_implementation.h index f0ad486e..294dd12d 100644 --- a/src/fallback/begin_implementation.h +++ b/src/fallback/begin_implementation.h @@ -2,3 +2,5 @@ #include "fallback/implementation.h" #define SIMDJSON_IMPLEMENTATION fallback + +#include "fallback/bitmanipulation.h" diff --git a/src/fallback/numberparsing.h b/src/fallback/numberparsing.h index 36586a21..ef5bb422 100644 --- a/src/fallback/numberparsing.h +++ b/src/fallback/numberparsing.h @@ -1,12 +1,6 @@ #ifndef SIMDJSON_FALLBACK_NUMBERPARSING_H #define SIMDJSON_FALLBACK_NUMBERPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "fallback/bitmanipulation.h" -#include -#include - #ifdef JSON_TEST_NUMBERS // for unit testing void found_invalid_number(const uint8_t *buf); void found_integer(int64_t result, const uint8_t *buf); diff --git a/src/fallback/stringparsing.h b/src/fallback/stringparsing.h index 0501ae5a..1359740f 100644 --- a/src/fallback/stringparsing.h +++ b/src/fallback/stringparsing.h @@ -2,7 +2,6 @@ #define SIMDJSON_FALLBACK_STRINGPARSING_H #include "simdjson.h" -#include "jsoncharutils.h" namespace simdjson { namespace fallback { diff --git a/src/generic/stage2/numberparsing.h b/src/generic/stage2/numberparsing.h index 14e8d957..74904c9a 100644 --- a/src/generic/stage2/numberparsing.h +++ b/src/generic/stage2/numberparsing.h @@ -1,3 +1,7 @@ +#include "jsoncharutils.h" +#include +#include + namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index c777cfe7..179074b2 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -1,7 +1,7 @@ // This file contains the common code every implementation uses // It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is include already includes -// "stringparsing.h" (this simplifies amalgation) + +#include "jsoncharutils.h" namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/haswell/begin_implementation.h b/src/haswell/begin_implementation.h index bfa87e29..f693c14a 100644 --- a/src/haswell/begin_implementation.h +++ b/src/haswell/begin_implementation.h @@ -1,6 +1,12 @@ #include "simdjson.h" + #include "haswell/implementation.h" #include "haswell/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION #define SIMDJSON_IMPLEMENTATION haswell SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt") + +#include "haswell/bitmanipulation.h" +#include "haswell/bitmask.h" +#include "haswell/simd.h" + diff --git a/src/haswell/bitmanipulation.h b/src/haswell/bitmanipulation.h index 24042649..78c56795 100644 --- a/src/haswell/bitmanipulation.h +++ b/src/haswell/bitmanipulation.h @@ -1,8 +1,6 @@ #ifndef SIMDJSON_HASWELL_BITMANIPULATION_H #define SIMDJSON_HASWELL_BITMANIPULATION_H -#include "simdjson.h" - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/haswell/bitmask.h b/src/haswell/bitmask.h index 9661f7ef..b29c6668 100644 --- a/src/haswell/bitmask.h +++ b/src/haswell/bitmask.h @@ -1,8 +1,6 @@ #ifndef SIMDJSON_HASWELL_BITMASK_H #define SIMDJSON_HASWELL_BITMASK_H -#include "simdjson.h" - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index 6e01afb5..05bced90 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -4,9 +4,6 @@ // // Stage 1 // -#include "haswell/bitmask.h" -#include "haswell/simd.h" -#include "haswell/bitmanipulation.h" namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/haswell/numberparsing.h b/src/haswell/numberparsing.h index 6bb0b0c2..e412adce 100644 --- a/src/haswell/numberparsing.h +++ b/src/haswell/numberparsing.h @@ -1,20 +1,6 @@ #ifndef SIMDJSON_HASWELL_NUMBERPARSING_H #define SIMDJSON_HASWELL_NUMBERPARSING_H -#include "simdjson.h" - -#include "jsoncharutils.h" -#include "haswell/bitmanipulation.h" -#include -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/haswell/simd.h b/src/haswell/simd.h index f2410842..3cd06800 100644 --- a/src/haswell/simd.h +++ b/src/haswell/simd.h @@ -1,9 +1,7 @@ #ifndef SIMDJSON_HASWELL_SIMD_H #define SIMDJSON_HASWELL_SIMD_H -#include "simdjson.h" #include "simdprune_tables.h" -#include "haswell/bitmanipulation.h" namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/haswell/stringparsing.h b/src/haswell/stringparsing.h index ce33f8e8..63b3a233 100644 --- a/src/haswell/stringparsing.h +++ b/src/haswell/stringparsing.h @@ -2,7 +2,6 @@ #define SIMDJSON_HASWELL_STRINGPARSING_H #include "simdjson.h" -#include "jsoncharutils.h" #include "haswell/simd.h" #include "haswell/bitmanipulation.h" diff --git a/src/westmere/begin_implementation.h b/src/westmere/begin_implementation.h index 8c7bfd16..54ddd2a9 100644 --- a/src/westmere/begin_implementation.h +++ b/src/westmere/begin_implementation.h @@ -4,3 +4,7 @@ #define SIMDJSON_IMPLEMENTATION westmere SIMDJSON_TARGET_REGION("sse4.2,pclmul") + +#include "westmere/bitmanipulation.h" +#include "westmere/bitmask.h" +#include "westmere/simd.h" diff --git a/src/westmere/bitmanipulation.h b/src/westmere/bitmanipulation.h index 3cf20323..a621381d 100644 --- a/src/westmere/bitmanipulation.h +++ b/src/westmere/bitmanipulation.h @@ -1,8 +1,6 @@ #ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H #define SIMDJSON_WESTMERE_BITMANIPULATION_H -#include "simdjson.h" - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/westmere/bitmask.h b/src/westmere/bitmask.h index d1da0eb1..79813fac 100644 --- a/src/westmere/bitmask.h +++ b/src/westmere/bitmask.h @@ -1,8 +1,6 @@ #ifndef SIMDJSON_WESTMERE_BITMASK_H #define SIMDJSON_WESTMERE_BITMASK_H -#include "simdjson.h" - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index 4192d2b9..c4322546 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -3,10 +3,6 @@ // // Stage 1 // -#include "westmere/bitmask.h" -#include "westmere/simd.h" -#include "westmere/bitmanipulation.h" -#include "westmere/implementation.h" namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/westmere/numberparsing.h b/src/westmere/numberparsing.h index ea9c0478..31b8bf5e 100644 --- a/src/westmere/numberparsing.h +++ b/src/westmere/numberparsing.h @@ -1,20 +1,6 @@ #ifndef SIMDJSON_WESTMERE_NUMBERPARSING_H #define SIMDJSON_WESTMERE_NUMBERPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "westmere/bitmanipulation.h" -#include -#include - - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/westmere/simd.h b/src/westmere/simd.h index 597e2fce..41eb3e27 100644 --- a/src/westmere/simd.h +++ b/src/westmere/simd.h @@ -1,9 +1,7 @@ #ifndef SIMDJSON_WESTMERE_SIMD_H #define SIMDJSON_WESTMERE_SIMD_H -#include "simdjson.h" #include "simdprune_tables.h" -#include "westmere/bitmanipulation.h" namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { diff --git a/src/westmere/stringparsing.h b/src/westmere/stringparsing.h index 1cc796d5..fa503316 100644 --- a/src/westmere/stringparsing.h +++ b/src/westmere/stringparsing.h @@ -1,11 +1,6 @@ #ifndef SIMDJSON_WESTMERE_STRINGPARSING_H #define SIMDJSON_WESTMERE_STRINGPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "westmere/simd.h" -#include "westmere/bitmanipulation.h" - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { From bf67c967d65623a76bd06c1873a2aed42ec314b2 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 16 Jul 2020 13:42:51 -0700 Subject: [PATCH 5/7] Inline jsoncharutils per-implementation --- src/arm64/dom_parser_implementation.cpp | 1 + src/fallback/dom_parser_implementation.cpp | 1 + src/generic/stage2/jsoncharutils.h | 134 ++++++++++++++++++ src/generic/stage2/numberparsing.h | 1 - src/generic/stage2/stringparsing.h | 2 - src/haswell/dom_parser_implementation.cpp | 1 + ...jsoncharutils.h => jsoncharutils_tables.h} | 118 +-------------- src/simdjson.cpp | 2 +- src/westmere/dom_parser_implementation.cpp | 2 + 9 files changed, 143 insertions(+), 119 deletions(-) create mode 100644 src/generic/stage2/jsoncharutils.h rename src/{jsoncharutils.h => jsoncharutils_tables.h} (92%) diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 0dee412b..50063149 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -1,5 +1,6 @@ #include "arm64/begin_implementation.h" #include "arm64/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 diff --git a/src/fallback/dom_parser_implementation.cpp b/src/fallback/dom_parser_implementation.cpp index 1df7d868..98034332 100644 --- a/src/fallback/dom_parser_implementation.cpp +++ b/src/fallback/dom_parser_implementation.cpp @@ -1,5 +1,6 @@ #include "fallback/begin_implementation.h" #include "fallback/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 diff --git a/src/generic/stage2/jsoncharutils.h b/src/generic/stage2/jsoncharutils.h new file mode 100644 index 00000000..e5f3755a --- /dev/null +++ b/src/generic/stage2/jsoncharutils.h @@ -0,0 +1,134 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace stage2 { + +// return non-zero if not a structural or whitespace char +// zero otherwise +really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) { + return structural_or_whitespace_or_null_negated[c]; +} + +// return non-zero if not a structural or whitespace char +// zero otherwise +really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return structural_or_whitespace_negated[c]; +} + +really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) { + return structural_or_whitespace_or_null[c]; +} + +really_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return structural_or_whitespace[c]; +} + +// returns a value with the high 16 bits set if not valid +// otherwise returns the conversion of the 4 hex digits at src into the bottom +// 16 bits of the 32-bit return register +// +// see +// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ +static inline uint32_t hex_to_u32_nocheck( + const uint8_t *src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = digit_to_val32[630 + src[0]]; + uint32_t v2 = digit_to_val32[420 + src[1]]; + uint32_t v3 = digit_to_val32[210 + src[2]]; + uint32_t v4 = digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; +} + +// given a code point cp, writes to c +// the utf-8 code, outputting the length in +// bytes, if the length is zero, the code point +// is invalid +// +// This can possibly be made faster using pdep +// and clz and table lookups, but JSON documents +// have few escaped code points, and the following +// function looks cheap. +// +// Note: we assume that surrogates are treated separately +// +inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r +} + +//// +// The following code is used in number parsing. It is not +// properly "char utils" stuff, but we move it here so that +// it does not get copied multiple times in the binaries (once +// per instruction set). +/// + +constexpr int FASTFLOAT_SMALLEST_POWER = -325; +constexpr int FASTFLOAT_LARGEST_POWER = 308; + +struct value128 { + uint64_t low; + uint64_t high; +}; + +#ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm +// this is a slow emulation routine for 32-bit +// +static inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; +} +static inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; +} +#endif + +really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { + value128 answer; +#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) +#ifdef _M_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // _M_ARM64 +#else // defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) + __uint128_t r = ((__uint128_t)value1) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; +} + +} // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/numberparsing.h b/src/generic/stage2/numberparsing.h index 74904c9a..6f1eb4ad 100644 --- a/src/generic/stage2/numberparsing.h +++ b/src/generic/stage2/numberparsing.h @@ -1,4 +1,3 @@ -#include "jsoncharutils.h" #include #include diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index 179074b2..ad144d5d 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -1,8 +1,6 @@ // This file contains the common code every implementation uses // It is intended to be included multiple times and compiled multiple times -#include "jsoncharutils.h" - namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index 05bced90..345b0150 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -1,5 +1,6 @@ #include "haswell/begin_implementation.h" #include "haswell/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 diff --git a/src/jsoncharutils.h b/src/jsoncharutils_tables.h similarity index 92% rename from src/jsoncharutils.h rename to src/jsoncharutils_tables.h index 4380db8a..365ef415 100644 --- a/src/jsoncharutils.h +++ b/src/jsoncharutils_tables.h @@ -1,5 +1,5 @@ -#ifndef SIMDJSON_JSONCHARUTILS_H -#define SIMDJSON_JSONCHARUTILS_H +#ifndef SIMDJSON_JSONCHARUTILS_TABLES_H +#define SIMDJSON_JSONCHARUTILS_TABLES_H #include "simdjson.h" @@ -34,12 +34,6 @@ const uint32_t structural_or_whitespace_or_null_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; -// return non-zero if not a structural or whitespace char -// zero otherwise -really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) { - return structural_or_whitespace_or_null_negated[c]; -} - const uint32_t structural_or_whitespace_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -57,12 +51,6 @@ const uint32_t structural_or_whitespace_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; -// return non-zero if not a structural or whitespace char -// zero otherwise -really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return structural_or_whitespace_negated[c]; -} - const uint32_t structural_or_whitespace_or_null[256] = { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, @@ -76,10 +64,6 @@ const uint32_t structural_or_whitespace_or_null[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) { - return structural_or_whitespace_or_null[c]; -} - const uint32_t structural_or_whitespace[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, @@ -93,10 +77,6 @@ const uint32_t structural_or_whitespace[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -really_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return structural_or_whitespace[c]; -} - const uint32_t digit_to_val32[886] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, @@ -246,62 +226,6 @@ const uint32_t digit_to_val32[886] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}; -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = digit_to_val32[630 + src[0]]; - uint32_t v2 = digit_to_val32[420 + src[1]]; - uint32_t v3 = digit_to_val32[210 + src[2]]; - uint32_t v4 = digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - //// // The following code is used in number parsing. It is not // properly "char utils" stuff, but we move it here so that @@ -317,42 +241,6 @@ struct value128 { uint64_t high; }; -#ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { - value128 answer; -#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) - __uint128_t r = ((__uint128_t)value1) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - // Precomputed powers of ten from 10^0 to 10^22. These // can be represented exactly using the double type. static const double power_of_ten[] = { @@ -1333,4 +1221,4 @@ const uint64_t mantissa_128[] = { } // namespace simdjson -#endif // SIMDJSON_JSONCHARUTILS_H +#endif // SIMDJSON_JSONCHARUTILS_TABLES_H diff --git a/src/simdjson.cpp b/src/simdjson.cpp index 84dc93a1..48af5147 100644 --- a/src/simdjson.cpp +++ b/src/simdjson.cpp @@ -9,7 +9,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS // Anything in the top level directory MUST be included outside of the #if statements // below, or amalgamation will screw them up! #include "isadetection.h" -#include "jsoncharutils.h" +#include "jsoncharutils_tables.h" #include "simdprune_tables.h" #if SIMDJSON_IMPLEMENTATION_ARM64 diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index c4322546..32c0fdd0 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -1,4 +1,6 @@ #include "westmere/begin_implementation.h" +#include "westmere/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 From a456d78fe01c2b5279461963a7be09200293f8f2 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 16 Jul 2020 13:56:29 -0700 Subject: [PATCH 6/7] really_inline more things --- src/generic/stage2/jsoncharutils.h | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/generic/stage2/jsoncharutils.h b/src/generic/stage2/jsoncharutils.h index e5f3755a..5a66bcda 100644 --- a/src/generic/stage2/jsoncharutils.h +++ b/src/generic/stage2/jsoncharutils.h @@ -49,7 +49,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -78,28 +78,13 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { return 0; // bad r } -//// -// The following code is used in number parsing. It is not -// properly "char utils" stuff, but we move it here so that -// it does not get copied multiple times in the binaries (once -// per instruction set). -/// - -constexpr int FASTFLOAT_SMALLEST_POWER = -325; -constexpr int FASTFLOAT_LARGEST_POWER = 308; - -struct value128 { - uint64_t low; - uint64_t high; -}; - #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static inline uint64_t __emulu(uint32_t x, uint32_t y) { +static really_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); From 7d347be9026572ab62dd4b6f130d6840e395e02a Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 16 Jul 2020 14:42:16 -0700 Subject: [PATCH 7/7] Untangle amalgamated headers --- singleheader/amalgamate.sh | 5 +++++ src/arm64/begin_implementation.h | 5 +---- src/fallback/begin_implementation.h | 5 +---- src/haswell/begin_implementation.h | 6 +++--- src/haswell/intrinsics.h | 11 +++-------- src/westmere/begin_implementation.h | 7 ++++--- 6 files changed, 17 insertions(+), 22 deletions(-) diff --git a/singleheader/amalgamate.sh b/singleheader/amalgamate.sh index a14f6a2b..b25f1f19 100755 --- a/singleheader/amalgamate.sh +++ b/singleheader/amalgamate.sh @@ -54,6 +54,11 @@ function doinclude() # generic includes are included multiple times if [[ "${file}" == *'generic/'*'.h' ]]; then dofile $AMALGAMATE_SOURCE_PATH $file + # begin/end_implementation are also included multiple times + elif [[ "${file}" == *'begin_implementation.h' ]]; then + dofile $AMALGAMATE_SOURCE_PATH $file + elif [[ "${file}" == *'end_implementation.h' ]]; then + dofile $AMALGAMATE_SOURCE_PATH $file elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then found_includes+=("$file") dofile $AMALGAMATE_SOURCE_PATH $file diff --git a/src/arm64/begin_implementation.h b/src/arm64/begin_implementation.h index 6cc981e6..b7c9fdab 100644 --- a/src/arm64/begin_implementation.h +++ b/src/arm64/begin_implementation.h @@ -1,9 +1,6 @@ -#include "simdjson.h" +#define SIMDJSON_IMPLEMENTATION arm64 #include "arm64/implementation.h" #include "arm64/intrinsics.h" - -#define SIMDJSON_IMPLEMENTATION arm64 - #include "arm64/bitmanipulation.h" #include "arm64/bitmask.h" #include "arm64/simd.h" diff --git a/src/fallback/begin_implementation.h b/src/fallback/begin_implementation.h index 294dd12d..dd2c6846 100644 --- a/src/fallback/begin_implementation.h +++ b/src/fallback/begin_implementation.h @@ -1,6 +1,3 @@ -#include "simdjson.h" -#include "fallback/implementation.h" - #define SIMDJSON_IMPLEMENTATION fallback - +#include "fallback/implementation.h" #include "fallback/bitmanipulation.h" diff --git a/src/haswell/begin_implementation.h b/src/haswell/begin_implementation.h index f693c14a..eaec9cf7 100644 --- a/src/haswell/begin_implementation.h +++ b/src/haswell/begin_implementation.h @@ -1,10 +1,10 @@ -#include "simdjson.h" +#define SIMDJSON_IMPLEMENTATION haswell +#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt") #include "haswell/implementation.h" #include "haswell/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION -#define SIMDJSON_IMPLEMENTATION haswell -SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt") +SIMDJSON_TARGET_HASWELL #include "haswell/bitmanipulation.h" #include "haswell/bitmask.h" diff --git a/src/haswell/intrinsics.h b/src/haswell/intrinsics.h index 69df3d36..c0fde426 100644 --- a/src/haswell/intrinsics.h +++ b/src/haswell/intrinsics.h @@ -41,17 +41,12 @@ // has it as a macro. #ifndef _blsr_u64 // we roll our own -#include "haswell/begin_implementation.h" -namespace simdjson { -namespace SIMDJSON_IMPLEMENTATION { +SIMDJSON_TARGET_HASWELL static really_inline uint64_t _blsr_u64(uint64_t n) { return (n - 1) & n; } -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson -#include "haswell/end_implementation.h" +SIMDJSON_UNTARGET_REGION #endif // _blsr_u64 -#endif - +#endif // SIMDJSON_CLANG_VISUAL_STUDIO #endif // SIMDJSON_HASWELL_INTRINSICS_H diff --git a/src/westmere/begin_implementation.h b/src/westmere/begin_implementation.h index 54ddd2a9..e500a0d9 100644 --- a/src/westmere/begin_implementation.h +++ b/src/westmere/begin_implementation.h @@ -1,9 +1,10 @@ -#include "simdjson.h" +#define SIMDJSON_IMPLEMENTATION westmere +#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul") + #include "westmere/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION #include "westmere/implementation.h" -#define SIMDJSON_IMPLEMENTATION westmere -SIMDJSON_TARGET_REGION("sse4.2,pclmul") +SIMDJSON_TARGET_WESTMERE #include "westmere/bitmanipulation.h" #include "westmere/bitmask.h"