diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h index e91a2423..d6a1b8d7 100644 --- a/include/simdjson/portability.h +++ b/include/simdjson/portability.h @@ -103,32 +103,27 @@ use a 64-bit target such as x64 or 64-bit ARM.") #ifdef __clang__ // clang does not have GCC push pop // warning: clang attribute push can't be used within a namespace in clang up -// til 8.0 so TARGET_REGION and UNTARGET_REGION must be *outside* of a +// til 8.0 so SIMDJSON_TARGET_REGION and SIMDJSON_UNTARGET_REGION must be *outside* of a // namespace. -#define TARGET_REGION(T) \ +#define SIMDJSON_TARGET_REGION(T) \ _Pragma(STRINGIFY( \ clang attribute push(__attribute__((target(T))), apply_to = function))) -#define UNTARGET_REGION _Pragma("clang attribute pop") +#define SIMDJSON_UNTARGET_REGION _Pragma("clang attribute pop") #elif defined(__GNUC__) // GCC is easier -#define TARGET_REGION(T) \ +#define SIMDJSON_TARGET_REGION(T) \ _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) -#define UNTARGET_REGION _Pragma("GCC pop_options") +#define SIMDJSON_UNTARGET_REGION _Pragma("GCC pop_options") #endif // clang then gcc #endif // x86 // Default target region macros don't do anything. -#ifndef TARGET_REGION -#define TARGET_REGION(T) -#define UNTARGET_REGION +#ifndef SIMDJSON_TARGET_REGION +#define SIMDJSON_TARGET_REGION(T) +#define SIMDJSON_UNTARGET_REGION #endif -// under GCC and CLANG, we use these two macros -#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul,lzcnt") -#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") -#define TARGET_ARM64 - // Is threading enabled? #if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT) #ifndef SIMDJSON_THREADS_ENABLED diff --git a/singleheader/amalgamate.sh b/singleheader/amalgamate.sh index a14f6a2b..b25f1f19 100755 --- a/singleheader/amalgamate.sh +++ b/singleheader/amalgamate.sh @@ -54,6 +54,11 @@ function doinclude() # generic includes are included multiple times if [[ "${file}" == *'generic/'*'.h' ]]; then dofile $AMALGAMATE_SOURCE_PATH $file + # begin/end_implementation are also included multiple times + elif [[ "${file}" == *'begin_implementation.h' ]]; then + dofile $AMALGAMATE_SOURCE_PATH $file + elif [[ "${file}" == *'end_implementation.h' ]]; then + dofile $AMALGAMATE_SOURCE_PATH $file elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then found_includes+=("$file") dofile $AMALGAMATE_SOURCE_PATH $file diff --git a/src/arm64/begin_implementation.h b/src/arm64/begin_implementation.h new file mode 100644 index 00000000..b7c9fdab --- /dev/null +++ b/src/arm64/begin_implementation.h @@ -0,0 +1,6 @@ +#define SIMDJSON_IMPLEMENTATION arm64 +#include "arm64/implementation.h" +#include "arm64/intrinsics.h" +#include "arm64/bitmanipulation.h" +#include "arm64/bitmask.h" +#include "arm64/simd.h" diff --git a/src/arm64/bitmanipulation.h b/src/arm64/bitmanipulation.h index fe002043..bb5a0afe 100644 --- a/src/arm64/bitmanipulation.h +++ b/src/arm64/bitmanipulation.h @@ -1,9 +1,6 @@ #ifndef SIMDJSON_ARM64_BITMANIPULATION_H #define SIMDJSON_ARM64_BITMANIPULATION_H -#include "simdjson.h" -#include "arm64/intrinsics.h" - namespace simdjson { namespace arm64 { diff --git a/src/arm64/bitmask.h b/src/arm64/bitmask.h index 3c8ceed9..8e953014 100644 --- a/src/arm64/bitmask.h +++ b/src/arm64/bitmask.h @@ -1,10 +1,6 @@ #ifndef SIMDJSON_ARM64_BITMASK_H #define SIMDJSON_ARM64_BITMASK_H -#include "simdjson.h" - -#include "arm64/intrinsics.h" - namespace simdjson { namespace arm64 { @@ -38,6 +34,6 @@ really_inline uint64_t prefix_xor(uint64_t bitmask) { } // namespace arm64 } // namespace simdjson -UNTARGET_REGION +SIMDJSON_UNTARGET_REGION #endif diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 5ac421d4..50063149 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -1,16 +1,12 @@ -#include "simdjson.h" -#include "arm64/implementation.h" +#include "arm64/begin_implementation.h" #include "arm64/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 // -#include "arm64/bitmask.h" -#include "arm64/simd.h" -#include "arm64/bitmanipulation.h" - namespace simdjson { -namespace arm64 { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -102,38 +98,12 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c return is_third_byte ^ is_fourth_byte; } -#include "generic/stage1/buf_block_reader.h" -#include "generic/stage1/json_string_scanner.h" -#include "generic/stage1/json_scanner.h" +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson -namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no - // benefit and therefore makes things worse. - // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} -} - -#include "generic/stage1/json_minifier.h" -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -#include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { - this->buf = _buf; - this->len = _len; - return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); -} #include "generic/stage1/utf8_validator.h" -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return simdjson::arm64::stage1::generic_validate_utf8(buf,len); -} -} // namespace arm64 -} // namespace simdjson // // Stage 2 @@ -141,20 +111,46 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons #include "arm64/stringparsing.h" #include "arm64/numberparsing.h" - -namespace simdjson { -namespace arm64 { - -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" +// +// Implementation-specific overrides +// +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + +namespace stage1 { + +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no + // benefit and therefore makes things worse. + // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} + +} // namespace stage1 + +WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { + return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); +} + +WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { + this->buf = _buf; + this->len = _len; + return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); +} + +WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return simdjson::arm64::stage1::generic_validate_utf8(buf,len); +} + WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); if (err) { return err; } return stage2(_doc); } -} // namespace arm64 +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson + +#include "arm64/end_implementation.h" diff --git a/src/arm64/dom_parser_implementation.h b/src/arm64/dom_parser_implementation.h index 29174f80..888b3422 100644 --- a/src/arm64/dom_parser_implementation.h +++ b/src/arm64/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace arm64 { - #include "generic/dom_parser_implementation.h" -} // namespace arm64 -} // namespace simdjson - #endif // SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/arm64/end_implementation.h b/src/arm64/end_implementation.h new file mode 100644 index 00000000..69c9f4b8 --- /dev/null +++ b/src/arm64/end_implementation.h @@ -0,0 +1 @@ +#undef SIMDJSON_IMPLEMENTATION diff --git a/src/arm64/implementation.cpp b/src/arm64/implementation.cpp index 7276aa29..a4b8958d 100644 --- a/src/arm64/implementation.cpp +++ b/src/arm64/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "arm64/implementation.h" +#include "arm64/begin_implementation.h" #include "arm64/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace arm64 { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,7 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace arm64 +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "arm64/end_implementation.h" \ No newline at end of file diff --git a/src/arm64/numberparsing.h b/src/arm64/numberparsing.h index 7355c687..d8cc2db0 100644 --- a/src/arm64/numberparsing.h +++ b/src/arm64/numberparsing.h @@ -1,21 +1,6 @@ #ifndef SIMDJSON_ARM64_NUMBERPARSING_H #define SIMDJSON_ARM64_NUMBERPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "arm64/intrinsics.h" -#include "arm64/bitmanipulation.h" -#include -#include - - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - namespace simdjson { namespace arm64 { @@ -29,11 +14,11 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); } +} // namespace arm64 +} // namespace simdjson + #define SWAR_NUMBER_PARSING #include "generic/stage2/numberparsing.h" -} // namespace arm64 -} // namespace simdjson - #endif // SIMDJSON_ARM64_NUMBERPARSING_H diff --git a/src/arm64/simd.h b/src/arm64/simd.h index c00ae99e..e3665427 100644 --- a/src/arm64/simd.h +++ b/src/arm64/simd.h @@ -4,7 +4,6 @@ #include "simdjson.h" #include "simdprune_tables.h" #include "arm64/bitmanipulation.h" -#include "arm64/intrinsics.h" #include diff --git a/src/arm64/stringparsing.h b/src/arm64/stringparsing.h index 0673e04c..d6e55d6e 100644 --- a/src/arm64/stringparsing.h +++ b/src/arm64/stringparsing.h @@ -2,9 +2,7 @@ #define SIMDJSON_ARM64_STRINGPARSING_H #include "simdjson.h" -#include "jsoncharutils.h" #include "arm64/simd.h" -#include "arm64/intrinsics.h" #include "arm64/bitmanipulation.h" namespace simdjson { @@ -45,9 +43,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 }; } -#include "generic/stage2/stringparsing.h" - } // namespace arm64 } // namespace simdjson +#include "generic/stage2/stringparsing.h" + #endif // SIMDJSON_ARM64_STRINGPARSING_H diff --git a/src/fallback/begin_implementation.h b/src/fallback/begin_implementation.h new file mode 100644 index 00000000..dd2c6846 --- /dev/null +++ b/src/fallback/begin_implementation.h @@ -0,0 +1,3 @@ +#define SIMDJSON_IMPLEMENTATION fallback +#include "fallback/implementation.h" +#include "fallback/bitmanipulation.h" diff --git a/src/fallback/dom_parser_implementation.cpp b/src/fallback/dom_parser_implementation.cpp index a474b68f..98034332 100644 --- a/src/fallback/dom_parser_implementation.cpp +++ b/src/fallback/dom_parser_implementation.cpp @@ -1,16 +1,16 @@ -#include "simdjson.h" -#include "fallback/implementation.h" +#include "fallback/begin_implementation.h" #include "fallback/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 // -namespace simdjson { -namespace fallback { -namespace stage1 { - #include "generic/stage1/find_next_document_index.h" +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace stage1 { + class structural_scanner { public: @@ -180,7 +180,6 @@ private: } // namespace stage1 - WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool partial) noexcept { this->buf = _buf; this->len = _len; @@ -308,7 +307,7 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons return true; } -} // namespace fallback +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson // @@ -316,14 +315,10 @@ WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) cons // #include "fallback/stringparsing.h" #include "fallback/numberparsing.h" +#include "generic/stage2/structural_parser.h" namespace simdjson { -namespace fallback { - -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" -#include "generic/stage2/structural_parser.h" +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); @@ -331,5 +326,7 @@ WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, siz return stage2(_doc); } -} // namespace fallback +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson + +#include "fallback/end_implementation.h" \ No newline at end of file diff --git a/src/fallback/dom_parser_implementation.h b/src/fallback/dom_parser_implementation.h index b648c36c..d7c83e07 100644 --- a/src/fallback/dom_parser_implementation.h +++ b/src/fallback/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace fallback { - #include "generic/dom_parser_implementation.h" -} // namespace fallback -} // namespace simdjson - #endif // SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/fallback/end_implementation.h b/src/fallback/end_implementation.h new file mode 100644 index 00000000..69c9f4b8 --- /dev/null +++ b/src/fallback/end_implementation.h @@ -0,0 +1 @@ +#undef SIMDJSON_IMPLEMENTATION diff --git a/src/fallback/implementation.cpp b/src/fallback/implementation.cpp index 3981f222..305e040c 100644 --- a/src/fallback/implementation.cpp +++ b/src/fallback/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "fallback/implementation.h" +#include "fallback/begin_implementation.h" #include "fallback/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace fallback { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,7 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace fallback +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "fallback/end_implementation.h" \ No newline at end of file diff --git a/src/fallback/implementation.h b/src/fallback/implementation.h index 5d92d385..5b99218c 100644 --- a/src/fallback/implementation.h +++ b/src/fallback/implementation.h @@ -26,7 +26,6 @@ public: }; } // namespace fallback - } // namespace simdjson #endif // SIMDJSON_FALLBACK_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/fallback/numberparsing.h b/src/fallback/numberparsing.h index 75760655..ef5bb422 100644 --- a/src/fallback/numberparsing.h +++ b/src/fallback/numberparsing.h @@ -1,12 +1,6 @@ #ifndef SIMDJSON_FALLBACK_NUMBERPARSING_H #define SIMDJSON_FALLBACK_NUMBERPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "fallback/bitmanipulation.h" -#include -#include - #ifdef JSON_TEST_NUMBERS // for unit testing void found_invalid_number(const uint8_t *buf); void found_integer(int64_t result, const uint8_t *buf); @@ -15,7 +9,7 @@ void found_float(double result, const uint8_t *buf); #endif namespace simdjson { -namespace fallback { +namespace SIMDJSON_IMPLEMENTATION { static really_inline uint32_t parse_eight_digits_unrolled(const char *chars) { uint32_t result = 0; for (int i=0;i<8;i++) { @@ -27,12 +21,10 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) return parse_eight_digits_unrolled((const char *)chars); } -#define SWAR_NUMBER_PARSING - -#include "generic/stage2/numberparsing.h" - -} // namespace fallback - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson +#define SWAR_NUMBER_PARSING +#include "generic/stage2/numberparsing.h" + #endif // SIMDJSON_FALLBACK_NUMBERPARSING_H diff --git a/src/fallback/stringparsing.h b/src/fallback/stringparsing.h index 599dc8ec..1359740f 100644 --- a/src/fallback/stringparsing.h +++ b/src/fallback/stringparsing.h @@ -2,7 +2,6 @@ #define SIMDJSON_FALLBACK_STRINGPARSING_H #include "simdjson.h" -#include "jsoncharutils.h" namespace simdjson { namespace fallback { @@ -27,9 +26,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 return { src[0] }; } -#include "generic/stage2/stringparsing.h" - } // namespace fallback } // namespace simdjson +#include "generic/stage2/stringparsing.h" + #endif // SIMDJSON_FALLBACK_STRINGPARSING_H diff --git a/src/generic/dom_parser_implementation.h b/src/generic/dom_parser_implementation.h index ca45b98e..ee59694b 100644 --- a/src/generic/dom_parser_implementation.h +++ b/src/generic/dom_parser_implementation.h @@ -1,3 +1,9 @@ +#include "simdjson.h" +#include "isadetection.h" + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // expectation: sizeof(scope_descriptor) = 64/8. struct scope_descriptor { uint32_t tape_index; // where, on the tape, does the scope ([,{) begins @@ -38,9 +44,15 @@ public: WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final; }; +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #include "generic/stage1/allocate.h" #include "generic/stage2/allocate.h" +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + really_inline dom_parser_implementation::dom_parser_implementation() {} // Leaving these here so they can be inlined if so desired @@ -57,3 +69,6 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth _max_depth = max_depth; return SUCCESS; } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/allocate.h b/src/generic/stage1/allocate.h index ab6fdb8a..e9ecb07e 100644 --- a/src/generic/stage1/allocate.h +++ b/src/generic/stage1/allocate.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { namespace allocate { @@ -15,3 +17,5 @@ really_inline error_code set_capacity(internal::dom_parser_implementation &parse } // namespace allocate } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/buf_block_reader.h b/src/generic/stage1/buf_block_reader.h index fb5c07e7..6eb28cc7 100644 --- a/src/generic/stage1/buf_block_reader.h +++ b/src/generic/stage1/buf_block_reader.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // Walks through a buffer in block-sized increments, loading the last part with spaces template struct buf_block_reader { @@ -81,3 +84,6 @@ template really_inline void buf_block_reader::advance() { idx += STEP_SIZE; } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/find_next_document_index.h b/src/generic/stage1/find_next_document_index.h index 41d6bdb1..7e42d7f3 100644 --- a/src/generic/stage1/find_next_document_index.h +++ b/src/generic/stage1/find_next_document_index.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + /** * This algorithm is used to quickly identify the last structural position that * makes up a complete document. @@ -84,3 +87,6 @@ really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left return len; } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_minifier.h b/src/generic/stage1/json_minifier.h index 4055d90a..46bfd4e5 100644 --- a/src/generic/stage1/json_minifier.h +++ b/src/generic/stage1/json_minifier.h @@ -3,6 +3,8 @@ // We assume the file in which it is included already includes // "simdjson/stage1.h" (this simplifies amalgation) +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { class json_minifier { @@ -75,3 +77,5 @@ error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, s } } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_scanner.h b/src/generic/stage1/json_scanner.h index 0834c4bd..8544a8cf 100644 --- a/src/generic/stage1/json_scanner.h +++ b/src/generic/stage1/json_scanner.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { /** @@ -102,3 +104,5 @@ really_inline error_code json_scanner::finish(bool streaming) { } } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_string_scanner.h b/src/generic/stage1/json_string_scanner.h index f37855c1..24d24d8c 100644 --- a/src/generic/stage1/json_string_scanner.h +++ b/src/generic/stage1/json_string_scanner.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { struct json_string_block { @@ -136,4 +138,6 @@ really_inline error_code json_string_scanner::finish(bool streaming) { return SUCCESS; } -} // namespace stage1 \ No newline at end of file +} // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index cf6ec261..c0883341 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -3,6 +3,14 @@ // We assume the file in which it is included already includes // "simdjson/stage1.h" (this simplifies amalgation) +#include "generic/stage1/buf_block_reader.h" +#include "generic/stage1/json_string_scanner.h" +#include "generic/stage1/json_scanner.h" +#include "generic/stage1/json_minifier.h" +#include "generic/stage1/find_next_document_index.h" + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { class bit_indexer { @@ -196,3 +204,5 @@ really_inline error_code json_structural_indexer::finish(dom_parser_implementati } } // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_fastvalidate_algorithm.h b/src/generic/stage1/utf8_fastvalidate_algorithm.h index ea6a88ce..fb9b95dc 100644 --- a/src/generic/stage1/utf8_fastvalidate_algorithm.h +++ b/src/generic/stage1/utf8_fastvalidate_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + /* * legal utf-8 byte sequence * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 @@ -176,3 +179,6 @@ struct utf8_checker { return this->has_error.any_bits_set_anywhere() ? simdjson::UTF8_ERROR : simdjson::SUCCESS; } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup2_algorithm.h b/src/generic/stage1/utf8_lookup2_algorithm.h index a5d7ca9a..6badfa07 100644 --- a/src/generic/stage1/utf8_lookup2_algorithm.h +++ b/src/generic/stage1/utf8_lookup2_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // // Detect Unicode errors. // @@ -216,3 +219,6 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup3_algorithm.h b/src/generic/stage1/utf8_lookup3_algorithm.h index e49cffcc..c8e439c2 100644 --- a/src/generic/stage1/utf8_lookup3_algorithm.h +++ b/src/generic/stage1/utf8_lookup3_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // // Detect Unicode errors. // @@ -236,3 +239,6 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup4_algorithm.h b/src/generic/stage1/utf8_lookup4_algorithm.h index 74573510..1bb0fcac 100644 --- a/src/generic/stage1/utf8_lookup4_algorithm.h +++ b/src/generic/stage1/utf8_lookup4_algorithm.h @@ -1,4 +1,5 @@ -using namespace simd; +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace utf8_validation { @@ -173,6 +174,9 @@ using namespace simd; } }; // struct utf8_checker -} +} // namespace utf8_validation -using utf8_validation::utf8_checker; \ No newline at end of file +using utf8_validation::utf8_checker; + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_lookup_algorithm.h b/src/generic/stage1/utf8_lookup_algorithm.h index ad2b97a5..4ad9b1fb 100644 --- a/src/generic/stage1/utf8_lookup_algorithm.h +++ b/src/generic/stage1/utf8_lookup_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + // // Detect Unicode errors. // @@ -297,3 +300,6 @@ struct utf8_checker { } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_range_algorithm.h b/src/generic/stage1/utf8_range_algorithm.h index 4caba6dc..ac301bb8 100644 --- a/src/generic/stage1/utf8_range_algorithm.h +++ b/src/generic/stage1/utf8_range_algorithm.h @@ -1,3 +1,6 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + /* * legal utf-8 byte sequence * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 @@ -178,3 +181,6 @@ struct utf8_checker { return this->has_error.any() ? simdjson::UTF8_ERROR : simdjson::SUCCESS; } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_validator.h b/src/generic/stage1/utf8_validator.h index cd05cd09..5e50166e 100644 --- a/src/generic/stage1/utf8_validator.h +++ b/src/generic/stage1/utf8_validator.h @@ -1,4 +1,7 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage1 { + /** * Validates that the string is actual UTF-8. */ @@ -23,4 +26,6 @@ bool generic_validate_utf8(const char * input, size_t length) { return generic_validate_utf8((const uint8_t *)input,length); } -} // namespace stage1 \ No newline at end of file +} // namespace stage1 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage1/utf8_zwegner_algorithm.h b/src/generic/stage1/utf8_zwegner_algorithm.h index eb9bd298..be81648c 100644 --- a/src/generic/stage1/utf8_zwegner_algorithm.h +++ b/src/generic/stage1/utf8_zwegner_algorithm.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { // // Detect UTF-8 errors. // @@ -358,3 +360,6 @@ struct utf8_checker { return (this->special_case_errors.any_bits_set_anywhere() | this->length_errors) ? simdjson::UTF8_ERROR : simdjson::SUCCESS; } }; // struct utf8_checker + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/allocate.h b/src/generic/stage2/allocate.h index ad35df11..e33a4847 100644 --- a/src/generic/stage2/allocate.h +++ b/src/generic/stage2/allocate.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace allocate { @@ -16,3 +18,5 @@ really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t } // namespace allocate } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/atomparsing.h b/src/generic/stage2/atomparsing.h index 2a23c525..c0d5f78f 100644 --- a/src/generic/stage2/atomparsing.h +++ b/src/generic/stage2/atomparsing.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace atomparsing { @@ -60,3 +62,5 @@ really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { } // namespace atomparsing } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/jsoncharutils.h b/src/generic/stage2/jsoncharutils.h new file mode 100644 index 00000000..5a66bcda --- /dev/null +++ b/src/generic/stage2/jsoncharutils.h @@ -0,0 +1,119 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace stage2 { + +// return non-zero if not a structural or whitespace char +// zero otherwise +really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) { + return structural_or_whitespace_or_null_negated[c]; +} + +// return non-zero if not a structural or whitespace char +// zero otherwise +really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return structural_or_whitespace_negated[c]; +} + +really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) { + return structural_or_whitespace_or_null[c]; +} + +really_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return structural_or_whitespace[c]; +} + +// returns a value with the high 16 bits set if not valid +// otherwise returns the conversion of the 4 hex digits at src into the bottom +// 16 bits of the 32-bit return register +// +// see +// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ +static inline uint32_t hex_to_u32_nocheck( + const uint8_t *src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = digit_to_val32[630 + src[0]]; + uint32_t v2 = digit_to_val32[420 + src[1]]; + uint32_t v3 = digit_to_val32[210 + src[2]]; + uint32_t v4 = digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; +} + +// given a code point cp, writes to c +// the utf-8 code, outputting the length in +// bytes, if the length is zero, the code point +// is invalid +// +// This can possibly be made faster using pdep +// and clz and table lookups, but JSON documents +// have few escaped code points, and the following +// function looks cheap. +// +// Note: we assume that surrogates are treated separately +// +really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r +} + +#ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm +// this is a slow emulation routine for 32-bit +// +static really_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; +} +static really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; +} +#endif + +really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { + value128 answer; +#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) +#ifdef _M_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // _M_ARM64 +#else // defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) + __uint128_t r = ((__uint128_t)value1) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; +} + +} // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/logger.h b/src/generic/stage2/logger.h index 2f543840..9da6c1ef 100644 --- a/src/generic/stage2/logger.h +++ b/src/generic/stage2/logger.h @@ -1,5 +1,7 @@ // This is for an internal-only stage 2 specific logger. // Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace logger { static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; @@ -61,4 +63,5 @@ namespace logger { } } } // namespace logger - +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/numberparsing.h b/src/generic/stage2/numberparsing.h index d6a6bc84..8a8fb390 100644 --- a/src/generic/stage2/numberparsing.h +++ b/src/generic/stage2/numberparsing.h @@ -1,3 +1,8 @@ +#include +#include + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace numberparsing { @@ -499,3 +504,5 @@ really_inline bool parse_number(const uint8_t *const src, W &writer) { } // namespace numberparsing } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/streaming_structural_parser.h b/src/generic/stage2/streaming_structural_parser.h deleted file mode 100755 index 8e63d028..00000000 --- a/src/generic/stage2/streaming_structural_parser.h +++ /dev/null @@ -1,168 +0,0 @@ -namespace stage2 { - -struct streaming_structural_parser: structural_parser { - really_inline streaming_structural_parser(dom_parser_implementation &_parser) : structural_parser(_parser, _parser.next_structural_index) {} - - // override to add streaming - WARN_UNUSED really_inline error_code start(ret_address_t finish_parser) { - // If there are no structurals left, return EMPTY - if (structurals.at_end(parser.n_structural_indexes)) { - return parser.error = EMPTY; - } - - log_start(); - init(); - - // Capacity ain't no thang for streaming, so we don't check it. - // Advance to the first character as soon as possible - advance_char(); - // Push the root scope (there is always at least one scope) - if (start_document(finish_parser)) { - return parser.error = DEPTH_ERROR; - } - return SUCCESS; - } - - // override to add streaming - WARN_UNUSED really_inline error_code finish() { - if ( structurals.past_end(parser.n_structural_indexes) ) { - log_error("IMPOSSIBLE: past the end of the JSON!"); - return parser.error = TAPE_ERROR; - } - end_document(); - parser.next_structural_index = uint32_t(structurals.next_structural_index()); - if (depth != 0) { - log_error("Unclosed objects or arrays!"); - return parser.error = TAPE_ERROR; - } - if (parser.containing_scope[depth].tape_index != 0) { - log_error("IMPOSSIBLE: root scope tape index did not start at 0!"); - return parser.error = TAPE_ERROR; - } - return SUCCESS; - } -}; - -} // namespace stage2 - -/************ - * The JSON is parsed to a tape, see the accompanying tape.md file - * for documentation. - ***********/ -WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - this->doc = &_doc; - static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); - stage2::streaming_structural_parser parser(*this); - error_code result = parser.start(addresses.finish); - if (result) { return result; } - // - // Read first value - // - switch (parser.structurals.current_char()) { - case '{': - FAIL_IF( parser.start_object(addresses.finish) ); - goto object_begin; - case '[': - FAIL_IF( parser.start_array(addresses.finish) ); - goto array_begin; - case '"': - FAIL_IF( parser.parse_string() ); - goto finish; - case 't': case 'f': case 'n': - FAIL_IF( parser.parse_single_atom() ); - goto finish; - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - FAIL_IF( - parser.structurals.with_space_terminated_copy([&](const uint8_t *copy, size_t idx) { - return parser.parse_number(©[idx], false); - }) - ); - goto finish; - case '-': - FAIL_IF( - parser.structurals.with_space_terminated_copy([&](const uint8_t *copy, size_t idx) { - return parser.parse_number(©[idx], true); - }) - ); - goto finish; - default: - parser.log_error("Document starts with a non-value character"); - goto error; - } - -// -// Object parser parsers -// -object_begin: - switch (parser.advance_char()) { - case '"': { - FAIL_IF( parser.parse_string(true) ); - goto object_key_parser; - } - case '}': - parser.end_object(); - goto scope_end; - default: - parser.log_error("Object does not start with a key"); - goto error; - } - -object_key_parser: - if (parser.advance_char() != ':' ) { parser.log_error("Missing colon after key in object"); goto error; } - parser.increment_count(); - parser.advance_char(); - GOTO( parser.parse_value(addresses, addresses.object_continue) ); - -object_continue: - switch (parser.advance_char()) { - case ',': - if (parser.advance_char() != '"' ) { parser.log_error("Key string missing at beginning of field in object"); goto error; } - FAIL_IF( parser.parse_string(true) ); - goto object_key_parser; - case '}': - parser.end_object(); - goto scope_end; - default: - parser.log_error("No comma between object fields"); - goto error; - } - -scope_end: - CONTINUE( parser.parser.ret_address[parser.depth] ); - -// -// Array parser parsers -// -array_begin: - if (parser.advance_char() == ']') { - parser.end_array(); - goto scope_end; - } - parser.increment_count(); - -main_array_switch: - /* we call update char on all paths in, so we can peek at parser.c on the - * on paths that can accept a close square brace (post-, and at start) */ - GOTO( parser.parse_value(addresses, addresses.array_continue) ); - -array_continue: - switch (parser.advance_char()) { - case ',': - parser.increment_count(); - parser.advance_char(); - goto main_array_switch; - case ']': - parser.end_array(); - goto scope_end; - default: - parser.log_error("Missing comma between array values"); - goto error; - } - -finish: - return parser.finish(); - -error: - return parser.error(); -} diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index 3c8b6419..ad144d5d 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -1,8 +1,8 @@ // This file contains the common code every implementation uses // It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is include already includes -// "stringparsing.h" (this simplifies amalgation) +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { namespace stringparsing { @@ -120,4 +120,6 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst } } // namespace stringparsing -} // namespace stage2 \ No newline at end of file +} // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/structural_iterator.h b/src/generic/stage2/structural_iterator.h index 2682b6d0..c1c7b759 100644 --- a/src/generic/stage2/structural_iterator.h +++ b/src/generic/stage2/structural_iterator.h @@ -1,3 +1,5 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { namespace stage2 { class structural_iterator { @@ -44,3 +46,5 @@ public: }; } // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/structural_parser.h b/src/generic/stage2/structural_parser.h index d6ca1aca..52e2ffa3 100644 --- a/src/generic/stage2/structural_parser.h +++ b/src/generic/stage2/structural_parser.h @@ -3,10 +3,16 @@ // We assume the file in which it is include already includes // "simdjson/stage2.h" (this simplifies amalgation) -namespace stage2 { -namespace { // Make everything here private - #include "generic/stage2/tape_writer.h" +#include "generic/stage2/logger.h" +#include "generic/stage2/atomparsing.h" +#include "generic/stage2/structural_iterator.h" + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace stage2 { + +namespace { // Make everything here private #ifdef SIMDJSON_USE_COMPUTED_GOTO #define INIT_ADDRESSES() { &&array_begin, &&array_continue, &&error, &&finish, &&object_begin, &&object_continue } @@ -489,3 +495,6 @@ WARN_UNUSED error_code dom_parser_implementation::stage2(dom::document &_doc) no WARN_UNUSED error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::parse_structurals(*this, _doc); } + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/generic/stage2/tape_writer.h b/src/generic/stage2/tape_writer.h index 47282d6a..aed7e91f 100644 --- a/src/generic/stage2/tape_writer.h +++ b/src/generic/stage2/tape_writer.h @@ -1,3 +1,7 @@ +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace stage2 { + struct tape_writer { /** The next place to write to tape */ uint64_t *next_tape_loc; @@ -93,3 +97,7 @@ really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_typ really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } + +} // namespace stage2 +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson diff --git a/src/haswell/begin_implementation.h b/src/haswell/begin_implementation.h new file mode 100644 index 00000000..eaec9cf7 --- /dev/null +++ b/src/haswell/begin_implementation.h @@ -0,0 +1,12 @@ +#define SIMDJSON_IMPLEMENTATION haswell +#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt") + +#include "haswell/implementation.h" +#include "haswell/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION + +SIMDJSON_TARGET_HASWELL + +#include "haswell/bitmanipulation.h" +#include "haswell/bitmask.h" +#include "haswell/simd.h" + diff --git a/src/haswell/bitmanipulation.h b/src/haswell/bitmanipulation.h index f6b8fc72..78c56795 100644 --- a/src/haswell/bitmanipulation.h +++ b/src/haswell/bitmanipulation.h @@ -1,13 +1,8 @@ #ifndef SIMDJSON_HASWELL_BITMANIPULATION_H #define SIMDJSON_HASWELL_BITMANIPULATION_H -#include "simdjson.h" - -#include "haswell/intrinsics.h" - -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { // We sometimes call trailing_zero on inputs that are zero, // but the algorithms do not end up using the returned value. @@ -73,8 +68,7 @@ really_inline bool mul_overflow(uint64_t value1, uint64_t value2, #endif } -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_BITMANIPULATION_H diff --git a/src/haswell/bitmask.h b/src/haswell/bitmask.h index c9ed2e0e..b29c6668 100644 --- a/src/haswell/bitmask.h +++ b/src/haswell/bitmask.h @@ -1,13 +1,8 @@ #ifndef SIMDJSON_HASWELL_BITMASK_H #define SIMDJSON_HASWELL_BITMASK_H -#include "simdjson.h" - -#include "haswell/intrinsics.h" - -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { // // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. @@ -22,9 +17,7 @@ really_inline uint64_t prefix_xor(const uint64_t bitmask) { return _mm_cvtsi128_si64(result); } -} // namespace haswell - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_BITMASK_H diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index 2a1f1891..345b0150 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -1,17 +1,13 @@ -#include "simdjson.h" -#include "haswell/implementation.h" +#include "haswell/begin_implementation.h" #include "haswell/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 // -#include "haswell/bitmask.h" -#include "haswell/simd.h" -#include "haswell/bitmanipulation.h" -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -68,60 +64,56 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson -#include "generic/stage1/buf_block_reader.h" -#include "generic/stage1/json_string_scanner.h" -#include "generic/stage1/json_scanner.h" - -namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} -} - -#include "generic/stage1/json_minifier.h" -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); -} - -#include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { - this->buf = _buf; - this->len = _len; - return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); -} #include "generic/stage1/utf8_validator.h" -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return simdjson::haswell::stage1::generic_validate_utf8(buf,len); -} -} // namespace haswell -} // namespace simdjson -UNTARGET_REGION // // Stage 2 // #include "haswell/stringparsing.h" #include "haswell/numberparsing.h" - -TARGET_HASWELL -namespace simdjson { -namespace haswell { - -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" +// +// Implementation-specific overrides +// +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + +namespace stage1 { + +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} + +} // namespace stage1 + +WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { + return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); +} + +WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { + this->buf = _buf; + this->len = _len; + return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); +} + +WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return simdjson::haswell::stage1::generic_validate_utf8(buf,len); +} + WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); if (err) { return err; } return stage2(_doc); } -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION + +#include "haswell/end_implementation.h" diff --git a/src/haswell/dom_parser_implementation.h b/src/haswell/dom_parser_implementation.h index fdf7d2b0..15d7808a 100644 --- a/src/haswell/dom_parser_implementation.h +++ b/src/haswell/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace haswell { - #include "generic/dom_parser_implementation.h" -} // namespace haswell -} // namespace simdjson - #endif // SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/haswell/end_implementation.h b/src/haswell/end_implementation.h new file mode 100644 index 00000000..cf78b56c --- /dev/null +++ b/src/haswell/end_implementation.h @@ -0,0 +1,2 @@ +#undef SIMDJSON_IMPLEMENTATION +SIMDJSON_UNTARGET_REGION diff --git a/src/haswell/implementation.cpp b/src/haswell/implementation.cpp index 53daa182..09c38aba 100644 --- a/src/haswell/implementation.cpp +++ b/src/haswell/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "haswell/implementation.h" +#include "haswell/begin_implementation.h" #include "haswell/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,8 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "haswell/end_implementation.h" + diff --git a/src/haswell/implementation.h b/src/haswell/implementation.h index 36e5f389..efd20e30 100644 --- a/src/haswell/implementation.h +++ b/src/haswell/implementation.h @@ -4,6 +4,7 @@ #include "simdjson.h" #include "isadetection.h" +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_REGION namespace simdjson { namespace haswell { diff --git a/src/haswell/intrinsics.h b/src/haswell/intrinsics.h index 5abc48c7..c0fde426 100644 --- a/src/haswell/intrinsics.h +++ b/src/haswell/intrinsics.h @@ -41,14 +41,12 @@ // has it as a macro. #ifndef _blsr_u64 // we roll our own -TARGET_HASWELL -static really_inline uint64_t simdjson_blsr_u64(uint64_t n) { +SIMDJSON_TARGET_HASWELL +static really_inline uint64_t _blsr_u64(uint64_t n) { return (n - 1) & n; } -UNTARGET_REGION -#define _blsr_u64(a) (simdjson_blsr_u64((a))) +SIMDJSON_UNTARGET_REGION #endif // _blsr_u64 -#endif - +#endif // SIMDJSON_CLANG_VISUAL_STUDIO #endif // SIMDJSON_HASWELL_INTRINSICS_H diff --git a/src/haswell/numberparsing.h b/src/haswell/numberparsing.h index 6f7e242e..e412adce 100644 --- a/src/haswell/numberparsing.h +++ b/src/haswell/numberparsing.h @@ -1,24 +1,9 @@ #ifndef SIMDJSON_HASWELL_NUMBERPARSING_H #define SIMDJSON_HASWELL_NUMBERPARSING_H -#include "simdjson.h" - -#include "jsoncharutils.h" -#include "haswell/intrinsics.h" -#include "haswell/bitmanipulation.h" -#include -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { + static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); @@ -37,13 +22,11 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) t4); // only captures the sum of the first 8 digits, drop the rest } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #define SWAR_NUMBER_PARSING #include "generic/stage2/numberparsing.h" -} // namespace haswell - -} // namespace simdjson -UNTARGET_REGION - #endif // SIMDJSON_HASWELL_NUMBERPARSING_H diff --git a/src/haswell/simd.h b/src/haswell/simd.h index c95f01df..3cd06800 100644 --- a/src/haswell/simd.h +++ b/src/haswell/simd.h @@ -1,14 +1,10 @@ #ifndef SIMDJSON_HASWELL_SIMD_H #define SIMDJSON_HASWELL_SIMD_H -#include "simdjson.h" #include "simdprune_tables.h" -#include "haswell/bitmanipulation.h" -#include "haswell/intrinsics.h" -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { namespace simd { // Forward-declared so they can be used by splat and friends. @@ -352,8 +348,7 @@ namespace simd { } // namespace simd -} // namespace haswell +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_HASWELL_SIMD_H diff --git a/src/haswell/stringparsing.h b/src/haswell/stringparsing.h index a8d7728b..63b3a233 100644 --- a/src/haswell/stringparsing.h +++ b/src/haswell/stringparsing.h @@ -2,14 +2,11 @@ #define SIMDJSON_HASWELL_STRINGPARSING_H #include "simdjson.h" -#include "jsoncharutils.h" #include "haswell/simd.h" -#include "haswell/intrinsics.h" #include "haswell/bitmanipulation.h" -TARGET_HASWELL namespace simdjson { -namespace haswell { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -41,10 +38,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 }; } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #include "generic/stage2/stringparsing.h" -} // namespace haswell -} // namespace simdjson -UNTARGET_REGION - #endif // SIMDJSON_HASWELL_STRINGPARSING_H diff --git a/src/jsoncharutils.h b/src/jsoncharutils_tables.h similarity index 92% rename from src/jsoncharutils.h rename to src/jsoncharutils_tables.h index 4380db8a..365ef415 100644 --- a/src/jsoncharutils.h +++ b/src/jsoncharutils_tables.h @@ -1,5 +1,5 @@ -#ifndef SIMDJSON_JSONCHARUTILS_H -#define SIMDJSON_JSONCHARUTILS_H +#ifndef SIMDJSON_JSONCHARUTILS_TABLES_H +#define SIMDJSON_JSONCHARUTILS_TABLES_H #include "simdjson.h" @@ -34,12 +34,6 @@ const uint32_t structural_or_whitespace_or_null_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; -// return non-zero if not a structural or whitespace char -// zero otherwise -really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) { - return structural_or_whitespace_or_null_negated[c]; -} - const uint32_t structural_or_whitespace_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -57,12 +51,6 @@ const uint32_t structural_or_whitespace_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; -// return non-zero if not a structural or whitespace char -// zero otherwise -really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return structural_or_whitespace_negated[c]; -} - const uint32_t structural_or_whitespace_or_null[256] = { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, @@ -76,10 +64,6 @@ const uint32_t structural_or_whitespace_or_null[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) { - return structural_or_whitespace_or_null[c]; -} - const uint32_t structural_or_whitespace[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, @@ -93,10 +77,6 @@ const uint32_t structural_or_whitespace[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -really_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return structural_or_whitespace[c]; -} - const uint32_t digit_to_val32[886] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, @@ -246,62 +226,6 @@ const uint32_t digit_to_val32[886] = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}; -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = digit_to_val32[630 + src[0]]; - uint32_t v2 = digit_to_val32[420 + src[1]]; - uint32_t v3 = digit_to_val32[210 + src[2]]; - uint32_t v4 = digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - //// // The following code is used in number parsing. It is not // properly "char utils" stuff, but we move it here so that @@ -317,42 +241,6 @@ struct value128 { uint64_t high; }; -#ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { - value128 answer; -#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) - __uint128_t r = ((__uint128_t)value1) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - // Precomputed powers of ten from 10^0 to 10^22. These // can be represented exactly using the double type. static const double power_of_ten[] = { @@ -1333,4 +1221,4 @@ const uint64_t mantissa_128[] = { } // namespace simdjson -#endif // SIMDJSON_JSONCHARUTILS_H +#endif // SIMDJSON_JSONCHARUTILS_TABLES_H diff --git a/src/simdjson.cpp b/src/simdjson.cpp index 84dc93a1..48af5147 100644 --- a/src/simdjson.cpp +++ b/src/simdjson.cpp @@ -9,7 +9,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS // Anything in the top level directory MUST be included outside of the #if statements // below, or amalgamation will screw them up! #include "isadetection.h" -#include "jsoncharutils.h" +#include "jsoncharutils_tables.h" #include "simdprune_tables.h" #if SIMDJSON_IMPLEMENTATION_ARM64 diff --git a/src/westmere/begin_implementation.h b/src/westmere/begin_implementation.h new file mode 100644 index 00000000..e500a0d9 --- /dev/null +++ b/src/westmere/begin_implementation.h @@ -0,0 +1,11 @@ +#define SIMDJSON_IMPLEMENTATION westmere +#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul") + +#include "westmere/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION +#include "westmere/implementation.h" + +SIMDJSON_TARGET_WESTMERE + +#include "westmere/bitmanipulation.h" +#include "westmere/bitmask.h" +#include "westmere/simd.h" diff --git a/src/westmere/bitmanipulation.h b/src/westmere/bitmanipulation.h index 29a8623a..a621381d 100644 --- a/src/westmere/bitmanipulation.h +++ b/src/westmere/bitmanipulation.h @@ -1,12 +1,8 @@ #ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H #define SIMDJSON_WESTMERE_BITMANIPULATION_H -#include "simdjson.h" -#include "westmere/intrinsics.h" - -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { // We sometimes call trailing_zero on inputs that are zero, // but the algorithms do not end up using the returned value. @@ -81,9 +77,7 @@ really_inline bool mul_overflow(uint64_t value1, uint64_t value2, #endif } -} // namespace westmere - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_BITMANIPULATION_H diff --git a/src/westmere/bitmask.h b/src/westmere/bitmask.h index 32bab6da..79813fac 100644 --- a/src/westmere/bitmask.h +++ b/src/westmere/bitmask.h @@ -1,12 +1,8 @@ #ifndef SIMDJSON_WESTMERE_BITMASK_H #define SIMDJSON_WESTMERE_BITMASK_H -#include "simdjson.h" -#include "westmere/intrinsics.h" - -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { // // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. @@ -21,9 +17,7 @@ really_inline uint64_t prefix_xor(const uint64_t bitmask) { return _mm_cvtsi128_si64(result); } -} // namespace westmere - +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_BITMASK_H diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index c665e76c..32c0fdd0 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -1,18 +1,13 @@ -#include "simdjson.h" -#include "westmere/implementation.h" +#include "westmere/begin_implementation.h" #include "westmere/dom_parser_implementation.h" +#include "generic/stage2/jsoncharutils.h" // // Stage 1 // -#include "westmere/bitmask.h" -#include "westmere/simd.h" -#include "westmere/bitmanipulation.h" -#include "westmere/implementation.h" -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -74,60 +69,56 @@ really_inline simd8 must_be_2_3_continuation(const simd8 prev2, c return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson -#include "generic/stage1/buf_block_reader.h" -#include "generic/stage1/json_string_scanner.h" -#include "generic/stage1/json_scanner.h" - -namespace stage1 { -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} -} - -#include "generic/stage1/json_minifier.h" -WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -#include "generic/stage1/find_next_document_index.h" #include "generic/stage1/utf8_lookup4_algorithm.h" #include "generic/stage1/json_structural_indexer.h" -WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { - this->buf = _buf; - this->len = _len; - return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); -} #include "generic/stage1/utf8_validator.h" -WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return simdjson::westmere::stage1::generic_validate_utf8(buf,len); -} -} // namespace westmere -} // namespace simdjson -UNTARGET_REGION // // Stage 2 // #include "westmere/stringparsing.h" #include "westmere/numberparsing.h" - -TARGET_WESTMERE -namespace simdjson { -namespace westmere { - -#include "generic/stage2/logger.h" -#include "generic/stage2/atomparsing.h" -#include "generic/stage2/structural_iterator.h" #include "generic/stage2/structural_parser.h" +// +// Implementation-specific overrides +// +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + +namespace stage1 { + +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} + +} // namespace stage1 + +WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { + return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); +} + +WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept { + this->buf = _buf; + this->len = _len; + return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); +} + +WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return simdjson::westmere::stage1::generic_validate_utf8(buf,len); +} + WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { error_code err = stage1(_buf, _len, false); if (err) { return err; } return stage2(_doc); } -} // namespace westmere +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION + +#include "westmere/end_implementation.h" diff --git a/src/westmere/dom_parser_implementation.h b/src/westmere/dom_parser_implementation.h index c36d9c8a..56e518b0 100644 --- a/src/westmere/dom_parser_implementation.h +++ b/src/westmere/dom_parser_implementation.h @@ -1,15 +1,6 @@ #ifndef SIMDJSON_WESTMERE_DOM_PARSER_IMPLEMENTATION_H #define SIMDJSON_WESTMERE_DOM_PARSER_IMPLEMENTATION_H -#include "simdjson.h" -#include "isadetection.h" - -namespace simdjson { -namespace westmere { - #include "generic/dom_parser_implementation.h" -} // namespace westmere -} // namespace simdjson - #endif // SIMDJSON_WESTMERE_DOM_PARSER_IMPLEMENTATION_H \ No newline at end of file diff --git a/src/westmere/end_implementation.h b/src/westmere/end_implementation.h new file mode 100644 index 00000000..cf78b56c --- /dev/null +++ b/src/westmere/end_implementation.h @@ -0,0 +1,2 @@ +#undef SIMDJSON_IMPLEMENTATION +SIMDJSON_UNTARGET_REGION diff --git a/src/westmere/implementation.cpp b/src/westmere/implementation.cpp index 56792a8c..e166210b 100644 --- a/src/westmere/implementation.cpp +++ b/src/westmere/implementation.cpp @@ -1,11 +1,8 @@ -#include "simdjson.h" -#include "westmere/implementation.h" +#include "westmere/begin_implementation.h" #include "westmere/dom_parser_implementation.h" -TARGET_HASWELL - namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { WARN_UNUSED error_code implementation::create_dom_parser_implementation( size_t capacity, @@ -19,7 +16,7 @@ WARN_UNUSED error_code implementation::create_dom_parser_implementation( return SUCCESS; } -} // namespace westmere +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION +#include "westmere/end_implementation.h" \ No newline at end of file diff --git a/src/westmere/implementation.h b/src/westmere/implementation.h index 69dceb20..3fe8e6ac 100644 --- a/src/westmere/implementation.h +++ b/src/westmere/implementation.h @@ -5,6 +5,7 @@ #include "simdjson/implementation.h" #include "isadetection.h" +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_REGION namespace simdjson { namespace westmere { diff --git a/src/westmere/numberparsing.h b/src/westmere/numberparsing.h index fa856e00..31b8bf5e 100644 --- a/src/westmere/numberparsing.h +++ b/src/westmere/numberparsing.h @@ -1,25 +1,9 @@ #ifndef SIMDJSON_WESTMERE_NUMBERPARSING_H #define SIMDJSON_WESTMERE_NUMBERPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "westmere/intrinsics.h" -#include "westmere/bitmanipulation.h" -#include -#include - - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - - -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { + static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); @@ -38,13 +22,11 @@ static really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) t4); // only captures the sum of the first 8 digits, drop the rest } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #define SWAR_NUMBER_PARSING #include "generic/stage2/numberparsing.h" -} // namespace westmere - -} // namespace simdjson -UNTARGET_REGION - #endif // SIMDJSON_WESTMERE_NUMBERPARSING_H diff --git a/src/westmere/simd.h b/src/westmere/simd.h index efbe508b..41eb3e27 100644 --- a/src/westmere/simd.h +++ b/src/westmere/simd.h @@ -1,16 +1,10 @@ #ifndef SIMDJSON_WESTMERE_SIMD_H #define SIMDJSON_WESTMERE_SIMD_H -#include "simdjson.h" #include "simdprune_tables.h" -#include "westmere/bitmanipulation.h" -#include "westmere/intrinsics.h" - - -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { namespace simd { template @@ -333,9 +327,7 @@ namespace simd { }; // struct simd8x64 } // namespace simd - -} // namespace westmere +} // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -UNTARGET_REGION #endif // SIMDJSON_WESTMERE_SIMD_INPUT_H diff --git a/src/westmere/stringparsing.h b/src/westmere/stringparsing.h index e2b0a4ac..fa503316 100644 --- a/src/westmere/stringparsing.h +++ b/src/westmere/stringparsing.h @@ -1,15 +1,8 @@ #ifndef SIMDJSON_WESTMERE_STRINGPARSING_H #define SIMDJSON_WESTMERE_STRINGPARSING_H -#include "simdjson.h" -#include "jsoncharutils.h" -#include "westmere/simd.h" -#include "westmere/intrinsics.h" -#include "westmere/bitmanipulation.h" - -TARGET_WESTMERE namespace simdjson { -namespace westmere { +namespace SIMDJSON_IMPLEMENTATION { using namespace simd; @@ -43,10 +36,9 @@ really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8 }; } +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + #include "generic/stage2/stringparsing.h" -} // namespace westmere -} // namespace simdjson -UNTARGET_REGION - #endif // SIMDJSON_WESTMERE_STRINGPARSING_H