Move ondemand implementation to include/

benchmark/bench_sax.cpp

@@ -6,7 +6,7 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 SIMDJSON_POP_DISABLE_WARNINGS
 
 #if SIMDJSON_IMPLEMENTATION_HASWELL
-#include "haswell/begin_implementation.h"
+#include "simdjson/haswell/begin.h"
 #endif // SIMDJSON_IMPLEMENTATION_HASWELL
 
 #include <iostream>
@@ -31,5 +31,5 @@ SIMDJSON_POP_DISABLE_WARNINGS
 BENCHMARK_MAIN();
 
 #if SIMDJSON_IMPLEMENTATION_HASWELL
-#include "haswell/end_implementation.h"
+#include "simdjson/haswell/end.h"
 #endif // SIMDJSON_IMPLEMENTATION_HASWELL

include/simdjson.h

@@ -72,6 +72,12 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
 #include "simdjson/internal/tape_ref-inl.h"
 #include "simdjson/dom/serialization-inl.h"
 
+// Implementations
+#include "simdjson/arm64.h"
+#include "simdjson/fallback.h"
+#include "simdjson/haswell.h"
+#include "simdjson/westmere.h"
+
 SIMDJSON_POP_DISABLE_WARNINGS
 
 #endif // SIMDJSON_H

include/simdjson/arm64.h

@@ -0,0 +1,24 @@
+#ifndef SIMDJSON_ARM64_H
+#define SIMDJSON_ARM64_H
+
+#include "simdjson/portability.h"
+
+#if SIMDJSON_IMPLEMENTATION_ARM64
+
+#include "simdjson/arm64/implementation.h"
+
+#include "simdjson/arm64/begin.h"
+#include "simdjson/generic/dom_parser_implementation.h"
+#include "simdjson/arm64/intrinsics.h"
+#include "simdjson/arm64/bitmanipulation.h"
+#include "simdjson/arm64/bitmask.h"
+#include "simdjson/arm64/simd.h"
+#include "simdjson/generic/jsoncharutils.h"
+#include "simdjson/generic/atomparsing.h"
+#include "simdjson/arm64/stringparsing.h"
+#include "simdjson/arm64/numberparsing.h"
+#include "simdjson/arm64/end.h"
+
+#endif // SIMDJSON_IMPLEMENTATION_ARM64
+
+#endif // SIMDJSON_ARM64_H

include/simdjson/arm64/begin.h

@@ -0,0 +1 @@
+#define SIMDJSON_IMPLEMENTATION arm64

include/simdjson/arm64/end.h

@@ -0,0 +1 @@
+#undef SIMDJSON_IMPLEMENTATION

include/simdjson/arm64/implementation.h

@@ -2,7 +2,7 @@
 #define SIMDJSON_ARM64_IMPLEMENTATION_H
 
 #include "simdjson.h"
-#include "isadetection.h"
+#include "simdjson/internal/isadetection.h"
 
 namespace simdjson {
 namespace arm64 {
@@ -12,7 +12,7 @@ using namespace simdjson::dom;
 
 class implementation final : public simdjson::implementation {
 public:
-  simdjson_really_inline implementation() : simdjson::implementation("arm64", "ARM NEON", instruction_set::NEON) {}
+  simdjson_really_inline implementation() : simdjson::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {}
   SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation(
     size_t capacity,
     size_t max_length,

include/simdjson/arm64/numberparsing.h

@@ -21,6 +21,6 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 
 #define SWAR_NUMBER_PARSING
 
-#include "generic/stage2/numberparsing.h"
+#include "simdjson/generic/numberparsing.h"
 
 #endif // SIMDJSON_ARM64_NUMBERPARSING_H

include/simdjson/arm64/simd.h

@@ -2,8 +2,8 @@
 #define SIMDJSON_ARM64_SIMD_H
 
 #include "simdjson.h"
-#include "simdprune_tables.h"
-#include "arm64/bitmanipulation.h"
+#include "simdjson/internal/simdprune_tables.h"
+#include "simdjson/arm64/bitmanipulation.h"
 #include <type_traits>
 
 

include/simdjson/arm64/stringparsing.h

@@ -2,8 +2,8 @@
 #define SIMDJSON_ARM64_STRINGPARSING_H
 
 #include "simdjson.h"
-#include "arm64/simd.h"
-#include "arm64/bitmanipulation.h"
+#include "simdjson/arm64/simd.h"
+#include "simdjson/arm64/bitmanipulation.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -48,6 +48,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "generic/stage2/stringparsing.h"
+#include "simdjson/generic/stringparsing.h"
 
 #endif // SIMDJSON_ARM64_STRINGPARSING_H

include/simdjson/common_defs.h

@@ -205,8 +205,8 @@ namespace std {
 #endif // SIMDJSON_HAS_STRING_VIEW
 #undef SIMDJSON_HAS_STRING_VIEW // We are not going to need this macro anymore.
 
-
-
+/// If EXPR is an error, returns it.
+#define SIMDJSON_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } }
 
 
 /**
@@ -253,5 +253,4 @@ namespace std {
  * End of the crazy locale headers.
  */
 
-
 #endif // SIMDJSON_COMMON_DEFS_H

include/simdjson/fallback.h

@@ -0,0 +1,20 @@
+#ifndef SIMDJSON_FALLBACK_H
+#define SIMDJSON_FALLBACK_H
+
+#include "simdjson/portability.h"
+
+#if SIMDJSON_IMPLEMENTATION_FALLBACK
+
+#include "simdjson/fallback/implementation.h"
+
+#include "simdjson/fallback/begin.h"
+#include "simdjson/generic/dom_parser_implementation.h"
+#include "simdjson/fallback/bitmanipulation.h"
+#include "simdjson/generic/jsoncharutils.h"
+#include "simdjson/generic/atomparsing.h"
+#include "simdjson/fallback/stringparsing.h"
+#include "simdjson/fallback/numberparsing.h"
+#include "simdjson/fallback/end.h"
+
+#endif // SIMDJSON_IMPLEMENTATION_FALLBACK
+#endif // SIMDJSON_FALLBACK_H

include/simdjson/fallback/begin.h

@@ -0,0 +1 @@
+#define SIMDJSON_IMPLEMENTATION fallback

include/simdjson/fallback/end.h

@@ -0,0 +1 @@
+#undef SIMDJSON_IMPLEMENTATION

include/simdjson/fallback/implementation.h

@@ -1,8 +1,7 @@
 #ifndef SIMDJSON_FALLBACK_IMPLEMENTATION_H
 #define SIMDJSON_FALLBACK_IMPLEMENTATION_H
 
-#include "simdjson.h"
-#include "isadetection.h"
+#include "simdjson/implementation.h"
 
 namespace simdjson {
 namespace fallback {

include/simdjson/fallback/numberparsing.h

@@ -27,6 +27,6 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace simdjson
 
 #define SWAR_NUMBER_PARSING
-#include "generic/stage2/numberparsing.h"
+#include "simdjson/generic/numberparsing.h"
 
 #endif // SIMDJSON_FALLBACK_NUMBERPARSING_H

include/simdjson/fallback/stringparsing.h

@@ -31,6 +31,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "generic/stage2/stringparsing.h"
+#include "simdjson/generic/stringparsing.h"
 
 #endif // SIMDJSON_FALLBACK_STRINGPARSING_H

include/simdjson/generic/atomparsing.h

@@ -1,7 +1,6 @@
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
-namespace stage2 {
 namespace atomparsing {
 
 // The string_to_uint32 is exclusively used to map literal strings to 32-bit values.
@@ -25,7 +24,7 @@ simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) {
 
 SIMDJSON_WARN_UNUSED
 simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) {
-  return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0;
+  return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
 }
 
 SIMDJSON_WARN_UNUSED
@@ -37,7 +36,7 @@ simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) {
 
 SIMDJSON_WARN_UNUSED
 simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) {
-  return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0;
+  return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0;
 }
 
 SIMDJSON_WARN_UNUSED
@@ -49,7 +48,7 @@ simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len)
 
 SIMDJSON_WARN_UNUSED
 simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) {
-  return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0;
+  return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
 }
 
 SIMDJSON_WARN_UNUSED
@@ -60,7 +59,6 @@ simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) {
 }
 
 } // namespace atomparsing
-} // namespace stage2
 } // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/dom_parser_implementation.h

@@ -1,5 +1,5 @@
 #include "simdjson.h"
-#include "isadetection.h"
+#include "simdjson/internal/isadetection.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -36,16 +36,16 @@ public:
   SIMDJSON_WARN_UNUSED error_code check_for_unclosed_array() noexcept;
   SIMDJSON_WARN_UNUSED error_code stage2(dom::document &doc) noexcept final;
   SIMDJSON_WARN_UNUSED error_code stage2_next(dom::document &doc) noexcept final;
-  SIMDJSON_WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final;
-  SIMDJSON_WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final;
+  inline SIMDJSON_WARN_UNUSED error_code set_capacity(size_t capacity) noexcept final;
+  inline SIMDJSON_WARN_UNUSED error_code set_max_depth(size_t max_depth) noexcept final;
+private:
+  simdjson_really_inline SIMDJSON_WARN_UNUSED error_code set_capacity_stage1(size_t capacity);
+
 };
 
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "generic/stage1/allocate.h"
-#include "generic/stage2/allocate.h"
-
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 
@@ -54,16 +54,24 @@ inline dom_parser_implementation::dom_parser_implementation(dom_parser_implement
 inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default;
 
 // Leaving these here so they can be inlined if so desired
-SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept {
-  error_code err = stage1::allocate::set_capacity(*this, capacity);
-  if (err) { _capacity = 0; return err; }
+inline SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept {
+  // Stage 1 index output
+  size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7;
+  structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] );
+  if (!structural_indexes) { _capacity = 0; return MEMALLOC; }
+  structural_indexes[0] = 0;
+  n_structural_indexes = 0;
+
   _capacity = capacity;
   return SUCCESS;
 }
 
-SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept {
-  error_code err = stage2::allocate::set_max_depth(*this, max_depth);
-  if (err) { _max_depth = 0; return err; }
+inline SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept {
+  // Stage 2 stacks
+  open_containers.reset(new (std::nothrow) open_container[max_depth]);
+  is_array.reset(new (std::nothrow) bool[max_depth]);
+  if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; }
+
   _max_depth = max_depth;
   return SUCCESS;
 }

include/simdjson/generic/jsoncharutils.h

@@ -1,16 +1,19 @@
+#include "simdjson/internal/jsoncharutils_tables.h"
+#include "simdjson/internal/numberparsing_tables.h"
+
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
-namespace stage2 {
+namespace jsoncharutils {
 
 // return non-zero if not a structural or whitespace char
 // zero otherwise
 simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) {
-  return structural_or_whitespace_negated[c];
+  return internal::structural_or_whitespace_negated[c];
 }
 
 simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) {
-  return structural_or_whitespace[c];
+  return internal::structural_or_whitespace[c];
 }
 
 // returns a value with the high 16 bits set if not valid
@@ -21,10 +24,10 @@ simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) {
 // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/
 static inline uint32_t hex_to_u32_nocheck(
     const uint8_t *src) { // strictly speaking, static inline is a C-ism
-  uint32_t v1 = digit_to_val32[630 + src[0]];
-  uint32_t v2 = digit_to_val32[420 + src[1]];
-  uint32_t v3 = digit_to_val32[210 + src[2]];
-  uint32_t v4 = digit_to_val32[0 + src[3]];
+  uint32_t v1 = internal::digit_to_val32[630 + src[0]];
+  uint32_t v2 = internal::digit_to_val32[420 + src[1]];
+  uint32_t v3 = internal::digit_to_val32[210 + src[2]];
+  uint32_t v4 = internal::digit_to_val32[0 + src[3]];
   return v1 | v2 | v3 | v4;
 }
 
@@ -87,6 +90,8 @@ static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64
 }
 #endif
 
+using internal::value128;
+
 simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) {
   value128 answer;
 #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS)
@@ -105,7 +110,7 @@ simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t va
   return answer;
 }
 
-} // namespace stage2
+} // namespace jsoncharutils
 } // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/numberparsing.h

@@ -1,12 +1,19 @@
+#include "simdjson/internal/numberparsing_tables.h"
 #include <cmath>
 #include <limits>
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
-namespace stage2 {
 namespace numberparsing {
 
+using internal::FASTFLOAT_LARGEST_POWER;
+using internal::FASTFLOAT_SMALLEST_POWER;
+using internal::value128;
+using internal::power_of_ten;
+using internal::mantissa_64;
+using internal::mantissa_128;
+
 #ifdef JSON_TEST_NUMBERS
 #define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR)
 #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
@@ -131,7 +138,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg
   // We want the most significant 64 bits of the product. We know
   // this will be non-zero because the most significant bit of i is
   // 1.
-  value128 product = full_multiplication(i, factor_mantissa);
+  value128 product = jsoncharutils::full_multiplication(i, factor_mantissa);
   uint64_t lower = product.low;
   uint64_t upper = product.high;
 
@@ -148,7 +155,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg
         mantissa_128[power - FASTFLOAT_SMALLEST_POWER];
     // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit
     // result (three 64-bit values)
-    product = full_multiplication(i, factor_mantissa_low);
+    product = jsoncharutils::full_multiplication(i, factor_mantissa_low);
     uint64_t product_low = product.low;
     uint64_t product_middle2 = product.high;
     uint64_t product_middle1 = lower;
@@ -498,7 +505,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
     SIMDJSON_TRY( parse_exponent(src, p, exponent) );
   }
   if (is_float) {
-    const bool clean_end = is_structural_or_whitespace(*p);
+    const bool clean_end = jsoncharutils::is_structural_or_whitespace(*p);
     SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) );
     if (!clean_end) { return INVALID_NUMBER(src); }
     return SUCCESS;
@@ -514,7 +521,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
       // Anything negative above INT64_MAX+1 is invalid
       if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src);  }
       WRITE_INTEGER(~i+1, src, writer);
-      if (!is_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
+      if (!jsoncharutils::is_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
       return SUCCESS;
     // Positive overflow check:
     // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
@@ -537,7 +544,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
   } else {
     WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
   }
-  if (!is_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
+  if (!jsoncharutils::is_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
   return SUCCESS;
 }
 
@@ -558,7 +565,7 @@ SIMDJSON_UNUSED simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
   // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
   int digit_count = int(p - start_digits);
   if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return NUMBER_ERROR; }
-  if (!is_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
+  if (!jsoncharutils::is_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
 
   // The longest positive 64-bit number is 20 digits.
   // We do it this way so we don't trigger this branch unless we must.
@@ -630,7 +637,7 @@ SIMDJSON_UNUSED simdjson_really_inline simdjson_result<uint64_t> parse_large_uns
     while (parse_digit(*p, i)) { p++; }
   }
 
-  if (!is_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
+  if (!jsoncharutils::is_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
   // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
   int digit_count = int(p - src);
   if (digit_count == 0 || ('0' == *src && digit_count > 1)) { return NUMBER_ERROR; }
@@ -656,7 +663,7 @@ SIMDJSON_UNUSED simdjson_really_inline simdjson_result<int64_t> parse_integer(co
   // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
   int digit_count = int(p - start_digits);
   if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return NUMBER_ERROR; }
-  if (!is_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
+  if (!jsoncharutils::is_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
 
   // The longest negative 64-bit number is 19 digits.
   // The longest positive 64-bit number is 20 digits.
@@ -748,7 +755,7 @@ SIMDJSON_UNUSED simdjson_really_inline simdjson_result<double> parse_double(cons
     overflow = overflow || exponent < FASTFLOAT_SMALLEST_POWER || exponent > FASTFLOAT_LARGEST_POWER;
   }
 
-  if (is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
+  if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
 
   //
   // Assemble (or slow-parse) the float
@@ -766,7 +773,6 @@ SIMDJSON_UNUSED simdjson_really_inline simdjson_result<double> parse_double(cons
 #endif // SIMDJSON_SKIPNUMBERPARSING
 
 } // namespace numberparsing
-} // namespace stage2
 } // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand-inl.h

@@ -0,0 +1,12 @@
+#include "simdjson/generic/ondemand/logger-inl.h"
+#include "simdjson/generic/ondemand/raw_json_string-inl.h"
+#include "simdjson/generic/ondemand/token_iterator-inl.h"
+#include "simdjson/generic/ondemand/json_iterator-inl.h"
+#include "simdjson/generic/ondemand/array_iterator-inl.h"
+#include "simdjson/generic/ondemand/object_iterator-inl.h"
+#include "simdjson/generic/ondemand/array-inl.h"
+#include "simdjson/generic/ondemand/document-inl.h"
+#include "simdjson/generic/ondemand/value-inl.h"
+#include "simdjson/generic/ondemand/field-inl.h"
+#include "simdjson/generic/ondemand/object-inl.h"
+#include "simdjson/generic/ondemand/parser-inl.h"

include/simdjson/generic/ondemand.h

@@ -0,0 +1,12 @@
+#include "simdjson/generic/ondemand/logger.h"
+#include "simdjson/generic/ondemand/raw_json_string.h"
+#include "simdjson/generic/ondemand/token_iterator.h"
+#include "simdjson/generic/ondemand/json_iterator.h"
+#include "simdjson/generic/ondemand/array_iterator.h"
+#include "simdjson/generic/ondemand/object_iterator.h"
+#include "simdjson/generic/ondemand/array.h"
+#include "simdjson/generic/ondemand/document.h"
+#include "simdjson/generic/ondemand/value.h"
+#include "simdjson/generic/ondemand/field.h"
+#include "simdjson/generic/ondemand/object.h"
+#include "simdjson/generic/ondemand/parser.h"

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -123,28 +123,28 @@ SIMDJSON_WARN_UNUSED simdjson_result<raw_json_string> json_iterator::get_raw_jso
 }
 SIMDJSON_WARN_UNUSED simdjson_result<uint64_t> json_iterator::get_uint64() noexcept {
   logger::log_value(*this, "uint64", "", 0);
-  return stage2::numberparsing::parse_unsigned(advance());
+  return numberparsing::parse_unsigned(advance());
 }
 SIMDJSON_WARN_UNUSED simdjson_result<int64_t> json_iterator::get_int64() noexcept {
   logger::log_value(*this, "int64", "", 0);
-  return stage2::numberparsing::parse_integer(advance());
+  return numberparsing::parse_integer(advance());
 }
 SIMDJSON_WARN_UNUSED simdjson_result<double> json_iterator::get_double() noexcept {
   logger::log_value(*this, "double", "", 0);
-  return stage2::numberparsing::parse_double(advance());
+  return numberparsing::parse_double(advance());
 }
 SIMDJSON_WARN_UNUSED simdjson_result<bool> json_iterator::get_bool() noexcept {
   logger::log_value(*this, "bool", "", 0);
   auto json = advance();
-  auto not_true = stage2::atomparsing::str4ncmp(json, "true");
-  auto not_false = stage2::atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
-  bool error = (not_true && not_false) || stage2::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
+  auto not_true = atomparsing::str4ncmp(json, "true");
+  auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
+  bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
   if (error) { logger::log_error(*this, "not a boolean"); return INCORRECT_TYPE; }
   return simdjson_result<bool>(!not_true);
 }
 simdjson_really_inline bool json_iterator::is_null() noexcept {
   auto json = peek();
-  if (!stage2::atomparsing::str4ncmp(json, "null")) {
+  if (!atomparsing::str4ncmp(json, "null")) {
     logger::log_value(*this, "null", "", 0);
     advance();
     return true;
@@ -158,7 +158,7 @@ SIMDJSON_WARN_UNUSED simdjson_really_inline bool json_iterator::advance_to_buffe
   auto len = peek_length();
   auto json = advance();
   if (len > N-1) {
-    if (stage2::is_not_structural_or_whitespace(json[N])) { return false; }
+    if (jsoncharutils::is_not_structural_or_whitespace(json[N])) { return false; }
     len = N-1;
   }
 
@@ -174,20 +174,20 @@ SIMDJSON_WARN_UNUSED simdjson_result<uint64_t> json_iterator::get_root_uint64()
   uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer
   if (!advance_to_buffer(tmpbuf)) { return NUMBER_ERROR; }
   logger::log_value(*this, "uint64", "", 0);
-  return stage2::numberparsing::parse_unsigned(buf);
+  return numberparsing::parse_unsigned(buf);
 }
 SIMDJSON_WARN_UNUSED simdjson_result<int64_t> json_iterator::get_root_int64() noexcept {
   uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer 
   if (!advance_to_buffer(tmpbuf)) { return NUMBER_ERROR; }
   logger::log_value(*this, "int64", "", 0);
-  return stage2::numberparsing::parse_integer(buf);
+  return numberparsing::parse_integer(buf);
 }
 SIMDJSON_WARN_UNUSED simdjson_result<double> json_iterator::get_root_double() noexcept {
   // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest number: -0.<fraction>e-308.
   uint8_t tmpbuf[1074+8+1];
   if (!advance_to_buffer(tmpbuf)) { return NUMBER_ERROR; }
   logger::log_value(*this, "double", "", 0);
-  return stage2::numberparsing::parse_double(buf);
+  return numberparsing::parse_double(buf);
 }
 SIMDJSON_WARN_UNUSED simdjson_result<bool> json_iterator::get_root_bool() noexcept {
   uint8_t tmpbuf[5+1];

include/simdjson/generic/ondemand/logger-inl.h

@@ -3,12 +3,20 @@ namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
 namespace logger {
 
+static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------";
 static constexpr const int LOG_EVENT_LEN = 20;
 static constexpr const int LOG_BUFFER_LEN = 30;
 static constexpr const int LOG_SMALL_BUFFER_LEN = 10;
-static int log_depth = 0;
-using SIMDJSON_IMPLEMENTATION::logger::DASHES;
-using SIMDJSON_IMPLEMENTATION::logger::printable_char;
+static int log_depth = 0; // Not threadsafe. Log only.
+
+// Helper to turn unprintable or newline characters into spaces
+static simdjson_really_inline char printable_char(char c) {
+  if (c >= 0x20) {
+    return c;
+  } else {
+    return ' ';
+  }
+}
 
 simdjson_really_inline void log_event(const json_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
   log_line(iter, "", type, detail, delta, depth_delta);

include/simdjson/generic/ondemand/logger.h

@@ -6,7 +6,11 @@ class json_iterator;
 
 namespace logger {
 
-static constexpr const bool LOG_ENABLED = SIMDJSON_IMPLEMENTATION::logger::LOG_ENABLED;
+#if SIMDJSON_VERBOSE_LOGGING
+  static constexpr const bool LOG_ENABLED = true;
+#else
+  static constexpr const bool LOG_ENABLED = false;
+#endif
 
 static simdjson_really_inline void log_headers() noexcept;
 static simdjson_really_inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;

include/simdjson/generic/ondemand/raw_json_string-inl.h

@@ -6,7 +6,7 @@ simdjson_really_inline raw_json_string::raw_json_string(const uint8_t * _buf) no
 
 simdjson_really_inline const char * raw_json_string::raw() const noexcept { return (const char *)buf; }
 simdjson_really_inline SIMDJSON_WARN_UNUSED simdjson_result<std::string_view> raw_json_string::unescape(uint8_t *&dst) const noexcept {
-  uint8_t *end = stage2::stringparsing::parse_string(buf, dst);
+  uint8_t *end = stringparsing::parse_string(buf, dst);
   if (!end) { return STRING_ERROR; }
   std::string_view result((const char *)dst, end-dst);
   dst = end;

include/simdjson/generic/ondemand/value-inl.h

@@ -68,7 +68,7 @@ simdjson_really_inline simdjson_result<double> value::get_double() && noexcept {
   iter.release(); // Communicate that we have handled the value PERF TODO elided, right?
   double result;
   error_code error;
-  if ((error = stage2::numberparsing::parse_double(json).get(result))) { log_error("not a double"); return error; }
+  if ((error = numberparsing::parse_double(json).get(result))) { log_error("not a double"); return error; }
   return result;
 }
 simdjson_really_inline simdjson_result<uint64_t> value::get_uint64() && noexcept {
@@ -76,7 +76,7 @@ simdjson_really_inline simdjson_result<uint64_t> value::get_uint64() && noexcept
   iter.release(); // Communicate that we have handled the value PERF TODO elided, right?
   uint64_t result;
   error_code error;
-  if ((error = stage2::numberparsing::parse_unsigned(json).get(result))) { log_error("not a unsigned integer"); return error; }
+  if ((error = numberparsing::parse_unsigned(json).get(result))) { log_error("not a unsigned integer"); return error; }
   return result;
 }
 simdjson_really_inline simdjson_result<int64_t> value::get_int64() && noexcept {
@@ -84,15 +84,15 @@ simdjson_really_inline simdjson_result<int64_t> value::get_int64() && noexcept {
   iter.release(); // Communicate that we have handled the value PERF TODO elided, right?
   int64_t result;
   error_code error;
-  if ((error = stage2::numberparsing::parse_integer(json).get(result))) { log_error("not an integer"); return error; }
+  if ((error = numberparsing::parse_integer(json).get(result))) { log_error("not an integer"); return error; }
   return result;
 }
 simdjson_really_inline simdjson_result<bool> value::get_bool() && noexcept {
   log_value("bool");
   iter.release(); // Communicate that we have handled the value PERF TODO elided, right?
-  auto not_true = stage2::atomparsing::str4ncmp(json, "true");
-  auto not_false = stage2::atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
-  bool error = (not_true && not_false) || stage2::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
+  auto not_true = atomparsing::str4ncmp(json, "true");
+  auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
+  bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
   if (error) { log_error("not a boolean"); return INCORRECT_TYPE; }
   return simdjson_result<bool>(!not_true, error ? INCORRECT_TYPE : SUCCESS);
 }
@@ -100,14 +100,14 @@ simdjson_really_inline bool value::is_null() & noexcept {
   log_value("null");
   // Since it's a *reference*, we may want to check something other than is_null() if it isn't null,
   // so we don't release the iterator unless it is actually null
-  if (stage2::atomparsing::str4ncmp(json, "null")) { return false; }
+  if (atomparsing::str4ncmp(json, "null")) { return false; }
   iter.release(); // Communicate that we have handled the value PERF TODO elided, right?
   return true;
 }
 simdjson_really_inline bool value::is_null() && noexcept {
   log_value("null");
   iter.release(); // Communicate that we have handled the value PERF TODO elided, right?
-  if (stage2::atomparsing::str4ncmp(json, "null")) { return false; }
+  if (atomparsing::str4ncmp(json, "null")) { return false; }
   return true;
 }
 

include/simdjson/generic/stringparsing.h

@@ -4,7 +4,6 @@
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
-namespace stage2 {
 namespace stringparsing {
 
 // begin copypasta
@@ -42,10 +41,10 @@ static const uint8_t escape_map[256] = {
 SIMDJSON_WARN_UNUSED
 simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
                                             uint8_t **dst_ptr) {
-  // hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
+  // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
   // conversion isn't valid; we defer the check for this to inside the
   // multilingual plane check
-  uint32_t code_point = hex_to_u32_nocheck(*src_ptr + 2);
+  uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
   *src_ptr += 6;
   // check for low surrogate for characters outside the Basic
   // Multilingual Plane.
@@ -53,7 +52,7 @@ simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
     if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') {
       return false;
     }
-    uint32_t code_point_2 = hex_to_u32_nocheck(*src_ptr + 2);
+    uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
 
     // if the first code point is invalid we will get here, as we will go past
     // the check for being outside the Basic Multilingual plane. If we don't
@@ -68,7 +67,7 @@ simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
         (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000;
     *src_ptr += 6;
   }
-  size_t offset = codepoint_to_utf8(code_point, *dst_ptr);
+  size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr);
   *dst_ptr += offset;
   return offset > 0;
 }
@@ -129,7 +128,6 @@ SIMDJSON_UNUSED SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_str
 }
 
 } // namespace stringparsing
-} // namespace stage2
 } // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/haswell.h

@@ -0,0 +1,40 @@
+#ifndef SIMDJSON_HASWELL_H
+#define SIMDJSON_HASWELL_H
+
+#include "simdjson/portability.h"
+
+#if SIMDJSON_IMPLEMENTATION_HASWELL
+
+#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt")
+
+//
+// These two need to be included outside SIMDJSON_TARGET_REGION
+//
+#include "simdjson/haswell/implementation.h"
+#include "simdjson/haswell/intrinsics.h"
+
+//
+// The rest need to be inside the region
+//
+#include "simdjson/haswell/begin.h"
+
+// Declarations
+#include "simdjson/generic/dom_parser_implementation.h"
+#include "simdjson/haswell/bitmanipulation.h"
+#include "simdjson/haswell/bitmask.h"
+#include "simdjson/haswell/simd.h"
+#include "simdjson/generic/jsoncharutils.h"
+#include "simdjson/generic/atomparsing.h"
+#include "simdjson/haswell/stringparsing.h"
+#include "simdjson/haswell/numberparsing.h"
+#include "simdjson/generic/implementation_simdjson_result_base.h"
+#include "simdjson/generic/ondemand.h"
+
+// Inline definitions
+#include "simdjson/generic/implementation_simdjson_result_base-inl.h"
+#include "simdjson/generic/ondemand-inl.h"
+
+#include "simdjson/haswell/end.h"
+
+#endif // SIMDJSON_IMPLEMENTATION_HASWELL
+#endif // SIMDJSON_HASWELL_COMMON_H

include/simdjson/haswell/begin.h

@@ -0,0 +1,2 @@
+#define SIMDJSON_IMPLEMENTATION haswell
+SIMDJSON_TARGET_HASWELL

include/simdjson/haswell/end.h

@@ -0,0 +1,2 @@
+SIMDJSON_UNTARGET_REGION
+#undef SIMDJSON_IMPLEMENTATION

include/simdjson/haswell/implementation.h

@@ -1,8 +1,7 @@
 #ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H
 #define SIMDJSON_HASWELL_IMPLEMENTATION_H
 
-#include "simdjson.h"
-#include "isadetection.h"
+#include "simdjson/implementation.h"
 
 // The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_REGION
 namespace simdjson {
@@ -15,7 +14,7 @@ public:
   simdjson_really_inline implementation() : simdjson::implementation(
       "haswell",
       "Intel/AMD AVX2",
-      instruction_set::AVX2 | instruction_set::PCLMULQDQ | instruction_set::BMI1 | instruction_set::BMI2
+      internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2
   ) {}
   SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation(
     size_t capacity,

include/simdjson/haswell/numberparsing.h

@@ -29,6 +29,6 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 
 #define SWAR_NUMBER_PARSING
 
-#include "generic/stage2/numberparsing.h"
+#include "simdjson/generic/numberparsing.h"
 
 #endif // SIMDJSON_HASWELL_NUMBERPARSING_H

include/simdjson/haswell/simd.h

@@ -1,7 +1,7 @@
 #ifndef SIMDJSON_HASWELL_SIMD_H
 #define SIMDJSON_HASWELL_SIMD_H
 
-#include "simdprune_tables.h"
+#include "simdjson/internal/simdprune_tables.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -120,6 +120,9 @@ namespace simd {
     // sensible, but the AVX ISA makes this kind of approach difficult.
     template<typename L>
     simdjson_really_inline void compress(uint32_t mask, L * output) const {
+      using internal::thintable_epi8;
+      using internal::BitsSetTable256mul2;
+      using internal::pshufb_combine_table;
       // this particular implementation was inspired by work done by @animetosho
       // we do it in four steps, first 8 bytes and then second 8 bytes...
       uint8_t mask1 = uint8_t(mask); // least significant 8 bits

include/simdjson/haswell/stringparsing.h

@@ -2,8 +2,8 @@
 #define SIMDJSON_HASWELL_STRINGPARSING_H
 
 #include "simdjson.h"
-#include "haswell/simd.h"
-#include "haswell/bitmanipulation.h"
+#include "simdjson/haswell/simd.h"
+#include "simdjson/haswell/bitmanipulation.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -43,6 +43,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "generic/stage2/stringparsing.h"
+#include "simdjson/generic/stringparsing.h"
 
 #endif // SIMDJSON_HASWELL_STRINGPARSING_H

include/simdjson/implementation.h

@@ -3,6 +3,7 @@
 
 #include "simdjson/common_defs.h"
 #include "simdjson/internal/dom_parser_implementation.h"
+#include "simdjson/internal/isadetection.h"
 #include <string>
 #include <atomic>
 #include <vector>
@@ -87,7 +88,7 @@ public:
    *
    * The instruction sets this implementation is compiled against.
    *
-   * @return a mask of all required `instruction_set` values
+   * @return a mask of all required `internal::instruction_set::` values
    */
   virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; };
 

include/simdjson/internal/isadetection.h

@@ -43,8 +43,8 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 */
 
-#ifndef SIMDJSON_ISADETECTION_H
-#define SIMDJSON_ISADETECTION_H
+#ifndef SIMDJSON_INTERNAL_ISADETECTION_H
+#define SIMDJSON_INTERNAL_ISADETECTION_H
 
 #include <cstdint>
 #include <cstdlib>
@@ -55,6 +55,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 namespace simdjson {
+namespace internal {
 
 
 enum instruction_set {
@@ -162,6 +163,7 @@ static inline uint32_t detect_supported_architectures() {
 
 #endif // end SIMD extension detection code
 
+} // namespace internal
 } // namespace simdjson
 
-#endif // SIMDJSON_ISADETECTION_H
+#endif // SIMDJSON_INTERNAL_ISADETECTION_H

include/simdjson/internal/jsoncharutils_tables.h

@@ -0,0 +1,35 @@
+#ifndef SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H
+#define SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H
+
+#include "simdjson.h"
+
+#ifdef JSON_TEST_STRINGS
+void found_string(const uint8_t *buf, const uint8_t *parsed_begin,
+                  const uint8_t *parsed_end);
+void found_bad_string(const uint8_t *buf);
+#endif
+
+namespace simdjson {
+namespace internal {
+
+constexpr int FASTFLOAT_SMALLEST_POWER = -325;
+constexpr int FASTFLOAT_LARGEST_POWER = 308;
+
+struct value128 {
+  uint64_t low;
+  uint64_t high;
+};
+
+// structural chars here are
+// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL)
+// we are also interested in the four whitespace characters
+// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
+
+extern SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace_negated[256];
+extern SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace[256];
+extern SIMDJSON_DLLIMPORTEXPORT const uint32_t digit_to_val32[886];
+
+} // namespace internal
+} // namespace simdjson
+
+#endif // SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H

include/simdjson/internal/numberparsing_tables.h

@@ -0,0 +1,27 @@
+#ifndef SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H
+#define SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H
+
+#include "simdjson.h"
+
+namespace simdjson {
+namespace internal {
+
+// Precomputed powers of ten from 10^0 to 10^22. These
+// can be represented exactly using the double type.
+extern SIMDJSON_DLLIMPORTEXPORT const double power_of_ten[];
+// The mantissas of powers of ten from -308 to 308, extended out to sixty four
+// bits. The array contains the powers of ten approximated
+// as a 64-bit mantissa. It goes from 10^FASTFLOAT_SMALLEST_POWER to 
+// 10^FASTFLOAT_LARGEST_POWER (inclusively). 
+// The mantissa is truncated, and
+// never rounded up. Uses about 5KB.
+extern SIMDJSON_DLLIMPORTEXPORT const uint64_t mantissa_64[];
+// A complement to mantissa_64
+// complete to a 128-bit mantissa.
+// Uses about 5KB but is rarely accessed.
+extern SIMDJSON_DLLIMPORTEXPORT const uint64_t mantissa_128[];
+
+} // namespace internal
+} // namespace simdjson
+
+#endif // SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H

include/simdjson/internal/simdprune_tables.h

@@ -0,0 +1,23 @@
+#ifndef SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H
+#define SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H
+
+#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE
+
+#include <cstdint>
+
+namespace simdjson { // table modified and copied from
+namespace internal { // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetTable
+
+extern SIMDJSON_DLLIMPORTEXPORT const unsigned char BitsSetTable256mul2[256];
+
+extern SIMDJSON_DLLIMPORTEXPORT const uint8_t pshufb_combine_table[272];
+
+// 256 * 8 bytes = 2kB, easily fits in cache.
+extern SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256];
+
+} // namespace internal
+} // namespace simdjson 
+
+#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE
+
+#endif // SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H

include/simdjson/westmere.h

@@ -0,0 +1,31 @@
+#ifndef SIMDJSON_WESTMERE_H
+#define SIMDJSON_WESTMERE_H
+
+#include "simdjson/portability.h"
+
+#if SIMDJSON_IMPLEMENTATION_WESTMERE
+
+#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul")
+
+//
+// These two need to be included outside SIMDJSON_TARGET_REGION
+//
+#include "simdjson/westmere/implementation.h"
+#include "simdjson/westmere/intrinsics.h"
+
+//
+// The rest need to be inside the region
+//
+#include "simdjson/westmere/begin.h"
+#include "simdjson/generic/dom_parser_implementation.h"
+#include "simdjson/westmere/bitmanipulation.h"
+#include "simdjson/westmere/bitmask.h"
+#include "simdjson/westmere/simd.h"
+#include "simdjson/generic/jsoncharutils.h"
+#include "simdjson/generic/atomparsing.h"
+#include "simdjson/westmere/stringparsing.h"
+#include "simdjson/westmere/numberparsing.h"
+#include "simdjson/westmere/end.h"
+
+#endif // SIMDJSON_IMPLEMENTATION_WESTMERE
+#endif // SIMDJSON_WESTMERE_COMMON_H

include/simdjson/westmere/begin.h

@@ -0,0 +1,2 @@
+#define SIMDJSON_IMPLEMENTATION westmere
+SIMDJSON_TARGET_WESTMERE

include/simdjson/westmere/end.h

@@ -0,0 +1,2 @@
+SIMDJSON_UNTARGET_REGION
+#undef SIMDJSON_IMPLEMENTATION

include/simdjson/westmere/implementation.h

@@ -1,21 +1,18 @@
 #ifndef SIMDJSON_WESTMERE_IMPLEMENTATION_H
 #define SIMDJSON_WESTMERE_IMPLEMENTATION_H
 
-#include "simdjson.h"
 #include "simdjson/implementation.h"
-#include "isadetection.h"
 
 // The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_REGION
 namespace simdjson {
 namespace westmere {
-namespace {
 
 using namespace simdjson;
 using namespace simdjson::dom;
 
 class implementation final : public simdjson::implementation {
 public:
-  simdjson_really_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", instruction_set::SSE42 | instruction_set::PCLMULQDQ) {}
+  simdjson_really_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {}
   SIMDJSON_WARN_UNUSED error_code create_dom_parser_implementation(
     size_t capacity,
     size_t max_length,
@@ -25,7 +22,6 @@ public:
   SIMDJSON_WARN_UNUSED bool validate_utf8(const char *buf, size_t len) const noexcept final;
 };
 
-} // unnamed namespace
 } // namespace westmere
 } // namespace simdjson
 

include/simdjson/westmere/numberparsing.h

@@ -29,6 +29,6 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 
 #define SWAR_NUMBER_PARSING
 
-#include "generic/stage2/numberparsing.h"
+#include "simdjson/generic/numberparsing.h"
 
 #endif //  SIMDJSON_WESTMERE_NUMBERPARSING_H

include/simdjson/westmere/simd.h

@@ -1,7 +1,7 @@
 #ifndef SIMDJSON_WESTMERE_SIMD_H
 #define SIMDJSON_WESTMERE_SIMD_H
 
-#include "simdprune_tables.h"
+#include "simdjson/internal/simdprune_tables.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -117,6 +117,9 @@ namespace simd {
     // sensible, but the AVX ISA makes this kind of approach difficult.
     template<typename L>
     simdjson_really_inline void compress(uint16_t mask, L * output) const {
+      using internal::thintable_epi8;
+      using internal::BitsSetTable256mul2;
+      using internal::pshufb_combine_table;
       // this particular implementation was inspired by work done by @animetosho
       // we do it in two steps, first 8 bytes and then second 8 bytes
       uint8_t mask1 = uint8_t(mask); // least significant 8 bits

include/simdjson/westmere/stringparsing.h

@@ -41,6 +41,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "generic/stage2/stringparsing.h"
+#include "simdjson/generic/stringparsing.h"
 
 #endif // SIMDJSON_WESTMERE_STRINGPARSING_H

src/arm64/begin_implementation.h

@@ -1,9 +0,0 @@
-#define SIMDJSON_IMPLEMENTATION arm64
-#define SIMDJSON_TARGET_IMPLEMENTATION
-#define SIMDJSON_UNTARGET_IMPLEMENTATION
-
-#include "arm64/implementation.h"
-#include "arm64/intrinsics.h"
-#include "arm64/bitmanipulation.h"
-#include "arm64/bitmask.h"
-#include "arm64/simd.h"

src/arm64/dom_parser_implementation.cpp

@@ -1,6 +1,4 @@
-#include "arm64/begin_implementation.h"
-#include "arm64/dom_parser_implementation.h"
-#include "generic/stage2/jsoncharutils.h"
+#include "simdjson/arm64/begin.h"
 
 //
 // Stage 1
@@ -111,8 +109,6 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 // Stage 2
 //
 
-#include "arm64/stringparsing.h"
-#include "arm64/numberparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -164,4 +160,4 @@ SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "arm64/end_implementation.h"
+#include "simdjson/arm64/end.h"

src/arm64/dom_parser_implementation.h

@@ -1,6 +0,0 @@
-#ifndef SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H
-#define SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H
-
-#include "generic/dom_parser_implementation.h"
-
-#endif // SIMDJSON_ARM64_DOM_PARSER_IMPLEMENTATION_H

src/arm64/end_implementation.h

@@ -1,3 +0,0 @@
-#undef SIMDJSON_IMPLEMENTATION
-#undef SIMDJSON_TARGET_IMPLEMENTATION
-#undef SIMDJSON_UNTARGET_IMPLEMENTATION

src/arm64/implementation.cpp

@@ -1,5 +1,4 @@
-#include "arm64/begin_implementation.h"
-#include "arm64/dom_parser_implementation.h"
+#include "simdjson/arm64/begin.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -19,4 +18,4 @@ SIMDJSON_WARN_UNUSED error_code implementation::create_dom_parser_implementation
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "arm64/end_implementation.h"
+#include "simdjson/arm64/end.h"

src/fallback/begin_implementation.h

@@ -1,6 +0,0 @@
-#define SIMDJSON_IMPLEMENTATION fallback
-#define SIMDJSON_TARGET_IMPLEMENTATION
-#define SIMDJSON_UNTARGET_IMPLEMENTATION
-
-#include "fallback/implementation.h"
-#include "fallback/bitmanipulation.h"

src/fallback/dom_parser_implementation.cpp

@@ -1,6 +1,4 @@
-#include "fallback/begin_implementation.h"
-#include "fallback/dom_parser_implementation.h"
-#include "generic/stage2/jsoncharutils.h"
+#include "simdjson/fallback/begin.h"
 
 //
 // Stage 1
@@ -315,8 +313,6 @@ SIMDJSON_WARN_UNUSED bool implementation::validate_utf8(const char *buf, size_t
 //
 // Stage 2
 //
-#include "fallback/stringparsing.h"
-#include "fallback/numberparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 namespace simdjson {
@@ -339,4 +335,4 @@ SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "fallback/end_implementation.h"
+#include "simdjson/fallback/end.h"

src/fallback/dom_parser_implementation.h

@@ -1,6 +0,0 @@
-#ifndef SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H
-#define SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H
-
-#include "generic/dom_parser_implementation.h"
-
-#endif // SIMDJSON_FALLBACK_DOM_PARSER_IMPLEMENTATION_H

src/fallback/end_implementation.h

@@ -1,3 +0,0 @@
-#undef SIMDJSON_IMPLEMENTATION
-#undef SIMDJSON_TARGET_IMPLEMENTATION
-#undef SIMDJSON_UNTARGET_IMPLEMENTATION

src/fallback/implementation.cpp

@@ -1,5 +1,4 @@
-#include "fallback/begin_implementation.h"
-#include "fallback/dom_parser_implementation.h"
+#include "simdjson/fallback/begin.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -19,4 +18,4 @@ SIMDJSON_WARN_UNUSED error_code implementation::create_dom_parser_implementation
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "fallback/end_implementation.h"
+#include "simdjson/fallback/end.h"

src/generic/ondemand.h

@@ -1,30 +0,0 @@
-// When *declaring* classes, we use host compiler options so it will all link.
-// When *defining* class methods, we use target compiler options.
-
-#include "generic/implementation_simdjson_result_base.h"
-#include "generic/ondemand/logger.h"
-#include "generic/ondemand/raw_json_string.h"
-#include "generic/ondemand/token_iterator.h"
-#include "generic/ondemand/json_iterator.h"
-#include "generic/ondemand/array_iterator.h"
-#include "generic/ondemand/object_iterator.h"
-#include "generic/ondemand/array.h"
-#include "generic/ondemand/document.h"
-#include "generic/ondemand/value.h"
-#include "generic/ondemand/field.h"
-#include "generic/ondemand/object.h"
-#include "generic/ondemand/parser.h"
-
-#include "generic/implementation_simdjson_result_base-inl.h"
-#include "generic/ondemand/logger-inl.h"
-#include "generic/ondemand/raw_json_string-inl.h"
-#include "generic/ondemand/token_iterator-inl.h"
-#include "generic/ondemand/json_iterator-inl.h"
-#include "generic/ondemand/array_iterator-inl.h"
-#include "generic/ondemand/object_iterator-inl.h"
-#include "generic/ondemand/array-inl.h"
-#include "generic/ondemand/document-inl.h"
-#include "generic/ondemand/value-inl.h"
-#include "generic/ondemand/field-inl.h"
-#include "generic/ondemand/object-inl.h"
-#include "generic/ondemand/parser-inl.h"

src/generic/stage1/allocate.h

@@ -1,23 +0,0 @@
-namespace simdjson {
-namespace SIMDJSON_IMPLEMENTATION {
-namespace {
-namespace stage1 {
-namespace allocate {
-
-//
-// Allocates stage 1 internal state and outputs in the parser
-//
-simdjson_really_inline error_code set_capacity(internal::dom_parser_implementation &parser, size_t capacity) {
-  size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7;
-  parser.structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] );
-  if (!parser.structural_indexes) { return MEMALLOC; }
-  parser.structural_indexes[0] = 0;
-  parser.n_structural_indexes = 0;
-  return SUCCESS;
-}
-
-} // namespace allocate
-} // namespace stage1
-} // unnamed namespace
-} // namespace SIMDJSON_IMPLEMENTATION
-} // namespace simdjson

src/generic/stage2/allocate.h

@@ -1,24 +0,0 @@
-namespace simdjson {
-namespace SIMDJSON_IMPLEMENTATION {
-namespace {
-namespace stage2 {
-namespace allocate {
-
-//
-// Allocates stage 2 internal state and outputs in the parser
-//
-simdjson_really_inline error_code set_max_depth(dom_parser_implementation &parser, size_t max_depth) {
-  parser.open_containers.reset(new (std::nothrow) open_container[max_depth]);
-  parser.is_array.reset(new (std::nothrow) bool[max_depth]);
-
-  if (!parser.is_array || !parser.open_containers) {
-    return MEMALLOC;
-  }
-  return SUCCESS;
-}
-
-} // namespace allocate
-} // namespace stage2
-} // unnamed namespace
-} // namespace SIMDJSON_IMPLEMENTATION
-} // namespace simdjson

src/generic/stage2/tape_builder.h

@@ -1,6 +1,5 @@
 #include "generic/stage2/json_iterator.h"
 #include "generic/stage2/tape_writer.h"
-#include "generic/stage2/atomparsing.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {

src/haswell/begin_implementation.h

@@ -1,17 +0,0 @@
-#ifndef SIMDJSON_TARGET_HASWELL
-#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt")
-#endif
-
-#define SIMDJSON_IMPLEMENTATION haswell
-#define SIMDJSON_TARGET_IMPLEMENTATION SIMDJSON_TARGET_HASWELL
-#define SIMDJSON_UNTARGET_IMPLEMENTATION SIMDJSON_UNTARGET_REGION
-
-#include "haswell/implementation.h"
-#include "haswell/intrinsics.h" // Generally need to be included outside SIMDJSON_TARGET_REGION
-
-SIMDJSON_TARGET_IMPLEMENTATION
-
-#include "haswell/bitmanipulation.h"
-#include "haswell/bitmask.h"
-#include "haswell/simd.h"
-

src/haswell/dom_parser_implementation.cpp

@@ -1,6 +1,4 @@
-#include "haswell/begin_implementation.h"
-#include "haswell/dom_parser_implementation.h"
-#include "generic/stage2/jsoncharutils.h"
+#include "simdjson/haswell/begin.h"
 
 //
 // Stage 1
@@ -114,8 +112,6 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 //
 // Stage 2
 //
-#include "haswell/stringparsing.h"
-#include "haswell/numberparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -165,6 +161,4 @@ SIMDJSON_WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "generic/ondemand.h"
-
-#include "haswell/end_implementation.h"
+#include "simdjson/haswell/end.h"

src/haswell/dom_parser_implementation.h

@@ -1,6 +0,0 @@
-#ifndef SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H
-#define SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H
-
-#include "generic/dom_parser_implementation.h"
-
-#endif // SIMDJSON_HASWELL_DOM_PARSER_IMPLEMENTATION_H

src/haswell/end_implementation.h

@@ -1,5 +0,0 @@
-SIMDJSON_UNTARGET_IMPLEMENTATION
-
-#undef SIMDJSON_IMPLEMENTATION
-#undef SIMDJSON_TARGET_IMPLEMENTATION
-#undef SIMDJSON_UNTARGET_IMPLEMENTATION

src/haswell/implementation.cpp

@@ -1,5 +1,4 @@
-#include "haswell/begin_implementation.h"
-#include "haswell/dom_parser_implementation.h"
+#include "simdjson/haswell/begin.h"
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
@@ -19,5 +18,5 @@ SIMDJSON_WARN_UNUSED error_code implementation::create_dom_parser_implementation
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "haswell/end_implementation.h"
+#include "simdjson/haswell/end.h"
 

src/implementation.cpp

@@ -1,44 +1,32 @@
 #include "simdjson.h"
-#include "isadetection.h"
-#include "simdprune_tables.h"
-
 #include <initializer_list>
 
-#define SIMDJSON_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } }
-
-// Static array of known implementations. We're hoping these get baked into the executable
-// without requiring a static initializer.
-
-#if SIMDJSON_IMPLEMENTATION_HASWELL
-#include "haswell/implementation.h"
-namespace simdjson { namespace internal { const haswell::implementation haswell_singleton{}; } }
-#endif // SIMDJSON_IMPLEMENTATION_HASWELL
-
-#if SIMDJSON_IMPLEMENTATION_WESTMERE
-#include "westmere/implementation.h"
-namespace simdjson { namespace internal { const westmere::implementation westmere_singleton{}; } }
-#endif // SIMDJSON_IMPLEMENTATION_WESTMERE
-
-#if SIMDJSON_IMPLEMENTATION_ARM64
-#include "arm64/implementation.h"
-namespace simdjson { namespace internal { const arm64::implementation arm64_singleton{}; } }
-#endif // SIMDJSON_IMPLEMENTATION_ARM64
-
-#if SIMDJSON_IMPLEMENTATION_FALLBACK
-#include "fallback/implementation.h"
-namespace simdjson { namespace internal { const fallback::implementation fallback_singleton{}; } }
-#endif // SIMDJSON_IMPLEMENTATION_FALLBACK
-
 namespace simdjson {
 
 bool implementation::supported_by_runtime_system() const {
   uint32_t required_instruction_sets = this->required_instruction_sets();
-  uint32_t supported_instruction_sets = detect_supported_architectures();
+  uint32_t supported_instruction_sets = internal::detect_supported_architectures();
   return ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets);
 }
 
 namespace internal {
 
+// Static array of known implementations. We're hoping these get baked into the executable
+// without requiring a static initializer.
+
+#if SIMDJSON_IMPLEMENTATION_HASWELL
+const haswell::implementation haswell_singleton{};
+#endif
+#if SIMDJSON_IMPLEMENTATION_WESTMERE
+const westmere::implementation westmere_singleton{};
+#endif // SIMDJSON_IMPLEMENTATION_WESTMERE
+#if SIMDJSON_IMPLEMENTATION_ARM64
+const arm64::implementation arm64_singleton{};
+#endif // SIMDJSON_IMPLEMENTATION_ARM64
+#if SIMDJSON_IMPLEMENTATION_FALLBACK
+const fallback::implementation fallback_singleton{};
+#endif // SIMDJSON_IMPLEMENTATION_FALLBACK
+
 /**
  * @private Detects best supported implementation on first use, and sets it
  */
@@ -121,7 +109,7 @@ const implementation * const *available_implementation_list::end() const noexcep
 }
 const implementation *available_implementation_list::detect_best_supported() const noexcept {
   // They are prelisted in priority order, so we just go down the list
-  uint32_t supported_instruction_sets = detect_supported_architectures();
+  uint32_t supported_instruction_sets = internal::detect_supported_architectures();
   for (const implementation *impl : internal::available_implementation_pointers) {
     uint32_t required_instruction_sets = impl->required_instruction_sets();
     if ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets) { return impl; }

src/internal/jsoncharutils_tables.cpp

@@ -0,0 +1,192 @@
+#include "simdjson.h"
+
+namespace simdjson {
+namespace internal {
+
+// structural chars here are
+// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL)
+// we are also interested in the four whitespace characters
+// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
+
+SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace_negated[256] = {
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
+
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1,
+
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+
+SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace[256] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+SIMDJSON_DLLIMPORTEXPORT const uint32_t digit_to_val32[886] = {
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x1,        0x2,        0x3,        0x4,        0x5,
+    0x6,        0x7,        0x8,        0x9,        0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa,
+    0xb,        0xc,        0xd,        0xe,        0xf,        0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa,        0xb,        0xc,        0xd,        0xe,
+    0xf,        0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x10,       0x20,       0x30,       0x40,       0x50,
+    0x60,       0x70,       0x80,       0x90,       0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa0,
+    0xb0,       0xc0,       0xd0,       0xe0,       0xf0,       0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa0,       0xb0,       0xc0,       0xd0,       0xe0,
+    0xf0,       0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x100,      0x200,      0x300,      0x400,      0x500,
+    0x600,      0x700,      0x800,      0x900,      0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa00,
+    0xb00,      0xc00,      0xd00,      0xe00,      0xf00,      0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa00,      0xb00,      0xc00,      0xd00,      0xe00,
+    0xf00,      0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x1000,     0x2000,     0x3000,     0x4000,     0x5000,
+    0x6000,     0x7000,     0x8000,     0x9000,     0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa000,
+    0xb000,     0xc000,     0xd000,     0xe000,     0xf000,     0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa000,     0xb000,     0xc000,     0xd000,     0xe000,
+    0xf000,     0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF};
+
+} // namespace internal
+} // namespace simdjson