jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity
simdjson/singleheader/simdjson.h

2670 lines
94 KiB
C++
Executable File
Raw Blame History

/* auto-generated on Sun Feb 2 15:10:09 PST 2020. Do not edit! */
/* begin file include/simdjson/simdjson_version.h */
// /include/simdjson/simdjson_version.h automatically generated by release.py,
// do not change by hand
#ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION
#define SIMDJSON_INCLUDE_SIMDJSON_VERSION
#define SIMDJSON_VERSION 0.2.1
namespace simdjson {
enum {
SIMDJSON_VERSION_MAJOR = 0,
SIMDJSON_VERSION_MINOR = 2,
SIMDJSON_VERSION_REVISION = 1
};
}
#endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION
/* end file include/simdjson/simdjson_version.h */
/* begin file include/simdjson/portability.h */
#ifndef SIMDJSON_PORTABILITY_H
#define SIMDJSON_PORTABILITY_H
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#ifdef _MSC_VER
#include <iso646.h>
#endif
#if defined(__x86_64__) || defined(_M_AMD64)
#define IS_X86_64 1
#endif
#if defined(__aarch64__) || defined(_M_ARM64)
#define IS_ARM64 1
#endif
// this is almost standard?
#undef STRINGIFY_IMPLEMENTATION_
#undef STRINGIFY
#define STRINGIFY_IMPLEMENTATION_(a) #a
#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a)
// we are going to use runtime dispatch
#ifdef IS_X86_64
#ifdef __clang__
// clang does not have GCC push pop
// warning: clang attribute push can't be used within a namespace in clang up
// til 8.0 so TARGET_REGION and UNTARGET_REGION must be *outside* of a
// namespace.
#define TARGET_REGION(T) \
_Pragma(STRINGIFY( \
clang attribute push(__attribute__((target(T))), apply_to = function)))
#define UNTARGET_REGION _Pragma("clang attribute pop")
#elif defined(__GNUC__)
// GCC is easier
#define TARGET_REGION(T) \
_Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T)))
#define UNTARGET_REGION _Pragma("GCC pop_options")
#endif // clang then gcc
#endif // x86
// Default target region macros don't do anything.
#ifndef TARGET_REGION
#define TARGET_REGION(T)
#define UNTARGET_REGION
#endif
// under GCC and CLANG, we use these two macros
#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul,lzcnt")
#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul")
#define TARGET_ARM64
// Threading is disabled
#undef SIMDJSON_THREADS_ENABLED
// Is threading enabled?
#if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT)
#define SIMDJSON_THREADS_ENABLED
#endif
#if defined(__clang__)
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
#elif defined(__GNUC__)
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
#else
#define NO_SANITIZE_UNDEFINED
#endif
#ifdef _MSC_VER
#include <intrin.h> // visual studio
#endif
#ifdef _MSC_VER
#define simdjson_strcasecmp _stricmp
#else
#define simdjson_strcasecmp strcasecmp
#endif
namespace simdjson {
// portable version of posix_memalign
static inline void *aligned_malloc(size_t alignment, size_t size) {
void *p;
#ifdef _MSC_VER
p = _aligned_malloc(size, alignment);
#elif defined(__MINGW32__) || defined(__MINGW64__)
p = __mingw_aligned_malloc(size, alignment);
#else
// somehow, if this is used before including "x86intrin.h", it creates an
// implicit defined warning.
if (posix_memalign(&p, alignment, size) != 0) {
return nullptr;
}
#endif
return p;
}
static inline char *aligned_malloc_char(size_t alignment, size_t size) {
return (char *)aligned_malloc(alignment, size);
}
static inline void aligned_free(void *mem_block) {
if (mem_block == nullptr) {
return;
}
#ifdef _MSC_VER
_aligned_free(mem_block);
#elif defined(__MINGW32__) || defined(__MINGW64__)
__mingw_aligned_free(mem_block);
#else
free(mem_block);
#endif
}
static inline void aligned_free_char(char *mem_block) {
aligned_free((void *)mem_block);
}
} // namespace simdjson
#endif // SIMDJSON_PORTABILITY_H
/* end file include/simdjson/portability.h */
/* begin file include/simdjson/isadetection.h */
/* From
https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h
Highly modified.
Copyright (c) 2016- Facebook, Inc (Adam Paszke)
Copyright (c) 2014- Facebook, Inc (Soumith Chintala)
Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu)
Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
Copyright (c) 2011-2013 NYU (Clement Farabet)
Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou,
Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute
(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert,
Samy Bengio, Johnny Mariethoz)
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories
America and IDIAP Research Institute nor the names of its contributors may be
used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef SIMDJSON_ISADETECTION_H
#define SIMDJSON_ISADETECTION_H
#include <stdint.h>
#include <stdlib.h>
#if defined(_MSC_VER)
#include <intrin.h>
#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
#include <cpuid.h>
#endif
namespace simdjson {
// Can be found on Intel ISA Reference for CPUID
constexpr uint32_t cpuid_avx2_bit = 1 << 5; // Bit 5 of EBX for EAX=0x7
constexpr uint32_t cpuid_bmi1_bit = 1 << 3; // bit 3 of EBX for EAX=0x7
constexpr uint32_t cpuid_bmi2_bit = 1 << 8; // bit 8 of EBX for EAX=0x7
constexpr uint32_t cpuid_sse42_bit = 1 << 20; // bit 20 of ECX for EAX=0x1
constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; // bit 1 of ECX for EAX=0x1
enum instruction_set {
DEFAULT = 0x0,
NEON = 0x1,
AVX2 = 0x4,
SSE42 = 0x8,
PCLMULQDQ = 0x10,
BMI1 = 0x20,
BMI2 = 0x40
};
#if defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64
#if defined(__ARM_NEON)
static inline uint32_t detect_supported_architectures() {
return instruction_set::NEON;
}
#else // ARM without NEON
static inline uint32_t detect_supported_architectures() {
return instruction_set::DEFAULT;
}
#endif
#else // x86
static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,
uint32_t *edx) {
#if defined(_MSC_VER)
int cpu_info[4];
__cpuid(cpu_info, *eax);
*eax = cpu_info[0];
*ebx = cpu_info[1];
*ecx = cpu_info[2];
*edx = cpu_info[3];
#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
uint32_t level = *eax;
__get_cpuid(level, eax, ebx, ecx, edx);
#else
uint32_t a = *eax, b, c = *ecx, d;
asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d));
*eax = a;
*ebx = b;
*ecx = c;
*edx = d;
#endif
}
static inline uint32_t detect_supported_architectures() {
uint32_t eax, ebx, ecx, edx;
uint32_t host_isa = 0x0;
// ECX for EAX=0x7
eax = 0x7;
ecx = 0x0;
cpuid(&eax, &ebx, &ecx, &edx);
#ifndef SIMDJSON_DISABLE_AVX2_DETECTION
if (ebx & cpuid_avx2_bit) {
host_isa |= instruction_set::AVX2;
}
#endif
if (ebx & cpuid_bmi1_bit) {
host_isa |= instruction_set::BMI1;
}
if (ebx & cpuid_bmi2_bit) {
host_isa |= instruction_set::BMI2;
}
// EBX for EAX=0x1
eax = 0x1;
cpuid(&eax, &ebx, &ecx, &edx);
if (ecx & cpuid_sse42_bit) {
host_isa |= instruction_set::SSE42;
}
if (ecx & cpuid_pclmulqdq_bit) {
host_isa |= instruction_set::PCLMULQDQ;
}
return host_isa;
}
#endif // end SIMD extension detection code
} // namespace simdjson
#endif
/* end file include/simdjson/isadetection.h */
/* begin file include/simdjson/jsonformatutils.h */
#ifndef SIMDJSON_JSONFORMATUTILS_H
#define SIMDJSON_JSONFORMATUTILS_H
#include <iomanip>
#include <iostream>
namespace simdjson {
// ends with zero char
static inline void print_with_escapes(const unsigned char *src,
std::ostream &os) {
while (*src) {
switch (*src) {
case '\b':
os << '\\';
os << 'b';
break;
case '\f':
os << '\\';
os << 'f';
break;
case '\n':
os << '\\';
os << 'n';
break;
case '\r':
os << '\\';
os << 'r';
break;
case '\"':
os << '\\';
os << '"';
break;
case '\t':
os << '\\';
os << 't';
break;
case '\\':
os << '\\';
os << '\\';
break;
default:
if (*src <= 0x1F) {
std::ios::fmtflags f(os.flags());
os << std::hex << std::setw(4) << std::setfill('0')
<< static_cast<int>(*src);
os.flags(f);
} else {
os << *src;
}
}
src++;
}
}
// print len chars
static inline void print_with_escapes(const unsigned char *src,
std::ostream &os, size_t len) {
const unsigned char *finalsrc = src + len;
while (src < finalsrc) {
switch (*src) {
case '\b':
os << '\\';
os << 'b';
break;
case '\f':
os << '\\';
os << 'f';
break;
case '\n':
os << '\\';
os << 'n';
break;
case '\r':
os << '\\';
os << 'r';
break;
case '\"':
os << '\\';
os << '"';
break;
case '\t':
os << '\\';
os << 't';
break;
case '\\':
os << '\\';
os << '\\';
break;
default:
if (*src <= 0x1F) {
std::ios::fmtflags f(os.flags());
os << std::hex << std::setw(4) << std::setfill('0')
<< static_cast<int>(*src);
os.flags(f);
} else {
os << *src;
}
}
src++;
}
}
static inline void print_with_escapes(const char *src, std::ostream &os) {
print_with_escapes(reinterpret_cast<const unsigned char *>(src), os);
}
static inline void print_with_escapes(const char *src, std::ostream &os,
size_t len) {
print_with_escapes(reinterpret_cast<const unsigned char *>(src), os, len);
}
} // namespace simdjson
#endif
/* end file include/simdjson/jsonformatutils.h */
/* begin file include/simdjson/simdjson.h */
#ifndef SIMDJSON_SIMDJSON_H
#define SIMDJSON_SIMDJSON_H
#ifndef __cplusplus
#error simdjson requires a C++ compiler
#endif
#ifndef SIMDJSON_CPLUSPLUS
#if defined(_MSVC_LANG) && !defined(__clang__)
#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG)
#else
#define SIMDJSON_CPLUSPLUS __cplusplus
#endif
#endif
#if (SIMDJSON_CPLUSPLUS < 201703L)
#error simdjson requires a compiler compliant with the C++17 standard
#endif
#include <string>
namespace simdjson {
// Represents the minimal architecture that would support an implementation
enum class Architecture {
UNSUPPORTED,
WESTMERE,
HASWELL,
ARM64,
// TODO remove 'native' in favor of runtime dispatch?
// the 'native' enum class value should point at a good default on the current
// machine
#ifdef IS_X86_64
NATIVE = WESTMERE
#elif defined(IS_ARM64)
NATIVE = ARM64
#endif
};
Architecture find_best_supported_architecture();
Architecture parse_architecture(char *architecture);
enum ErrorValues {
SUCCESS = 0,
SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data
CAPACITY, // This ParsedJson can't support a document that big
MEMALLOC, // Error allocating memory, most likely out of memory
TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this
// is a generic error
DEPTH_ERROR, // Your document exceeds the user-specified depth limitation
STRING_ERROR, // Problem while parsing a string
T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't'
F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f'
N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n'
NUMBER_ERROR, // Problem while parsing a number
UTF8_ERROR, // the input is not valid UTF-8
UNINITIALIZED, // unknown error, or uninitialized document
EMPTY, // no structural element found
UNESCAPED_CHARS, // found unescaped characters in a string.
UNCLOSED_STRING, // missing quote at the end
UNEXPECTED_ERROR // indicative of a bug in simdjson
};
const std::string &error_message(const int);
} // namespace simdjson
#endif // SIMDJSON_SIMDJSON_H
/* end file include/simdjson/simdjson.h */
/* begin file include/simdjson/common_defs.h */
#ifndef SIMDJSON_COMMON_DEFS_H
#define SIMDJSON_COMMON_DEFS_H
#include <cassert>
// we support documents up to 4GB
#define SIMDJSON_MAXSIZE_BYTES 0xFFFFFFFF
// the input buf should be readable up to buf + SIMDJSON_PADDING
// this is a stopgap; there should be a better description of the
// main loop and its behavior that abstracts over this
// See https://github.com/lemire/simdjson/issues/174
#define SIMDJSON_PADDING 32
#if defined(__GNUC__)
// Marks a block with a name so that MCA analysis can see it.
#define BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name);
#define END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name);
#define DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name);
#else
#define BEGIN_DEBUG_BLOCK(name)
#define END_DEBUG_BLOCK(name)
#define DEBUG_BLOCK(name, block)
#endif
#if !defined(_MSC_VER) && !defined(SIMDJSON_NO_COMPUTED_GOTO)
// Implemented using Labels as Values which works in GCC and CLANG (and maybe
// also in Intel's compiler), but won't work in MSVC.
#define SIMDJSON_USE_COMPUTED_GOTO
#endif
// Align to N-byte boundary
#define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
#define ROUNDDOWN_N(a, n) ((a) & ~((n)-1))
#define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
#ifdef _MSC_VER
#define really_inline __forceinline
#define never_inline __declspec(noinline)
#define UNUSED
#define WARN_UNUSED
#ifndef likely
#define likely(x) x
#endif
#ifndef unlikely
#define unlikely(x) x
#endif
#else
#define really_inline inline __attribute__((always_inline, unused))
#define never_inline inline __attribute__((noinline, unused))
#define UNUSED __attribute__((unused))
#define WARN_UNUSED __attribute__((warn_unused_result))
#ifndef likely
#define likely(x) __builtin_expect(!!(x), 1)
#endif
#ifndef unlikely
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
#endif // MSC_VER
#endif // SIMDJSON_COMMON_DEFS_H
/* end file include/simdjson/common_defs.h */
/* begin file include/simdjson/padded_string.h */
#ifndef SIMDJSON_PADDING_STRING_H
#define SIMDJSON_PADDING_STRING_H
#include <cstring>
#include <memory>
#include <string>
namespace simdjson {
// low-level function to allocate memory with padding so we can read past the
// "length" bytes safely. if you must provide a pointer to some data, create it
// with this function: length is the max. size in bytes of the string caller is
// responsible to free the memory (free(...))
inline char *allocate_padded_buffer(size_t length) noexcept {
// we could do a simple malloc
// return (char *) malloc(length + SIMDJSON_PADDING);
// However, we might as well align to cache lines...
size_t totalpaddedlength = length + SIMDJSON_PADDING;
char *padded_buffer = aligned_malloc_char(64, totalpaddedlength);
#ifndef NDEBUG
if (padded_buffer == nullptr) {
errno = EINVAL;
perror("simdjson::allocate_padded_buffer() aligned_malloc_char() failed");
return nullptr;
}
#endif // NDEBUG
memset(padded_buffer + length, 0, totalpaddedlength - length);
return padded_buffer;
} // allocate_padded_buffer
// Simple string with padded allocation.
// We deliberately forbid copies, users should rely on swap or move
// constructors.
struct padded_string final {
explicit padded_string() noexcept : viable_size(0), data_ptr(nullptr) {}
explicit padded_string(size_t length) noexcept
: viable_size(length), data_ptr(allocate_padded_buffer(length)) {
if (data_ptr != nullptr)
data_ptr[length] = '\0'; // easier when you need a c_str
}
explicit padded_string(const char *data, size_t length) noexcept
: viable_size(length), data_ptr(allocate_padded_buffer(length)) {
if ((data != nullptr) and (data_ptr != nullptr)) {
memcpy(data_ptr, data, length);
data_ptr[length] = '\0'; // easier when you need a c_str
}
}
// note: do not pass std::string arguments by value
padded_string(const std::string & str_ ) noexcept
: viable_size(str_.size()), data_ptr(allocate_padded_buffer(str_.size())) {
if (data_ptr != nullptr) {
memcpy(data_ptr, str_.data(), str_.size());
data_ptr[str_.size()] = '\0'; // easier when you need a c_str
}
}
// note: do pass std::string_view arguments by value
padded_string(std::string_view sv_) noexcept
: viable_size(sv_.size()), data_ptr(allocate_padded_buffer(sv_.size())) {
if (data_ptr != nullptr) {
memcpy(data_ptr, sv_.data(), sv_.size());
data_ptr[sv_.size()] = '\0'; // easier when you need a c_str
}
}
padded_string(padded_string &&o) noexcept
: viable_size(o.viable_size), data_ptr(o.data_ptr) {
o.data_ptr = nullptr; // we take ownership
}
padded_string &operator=(padded_string &&o) {
aligned_free_char(data_ptr);
data_ptr = o.data_ptr;
viable_size = o.viable_size;
o.data_ptr = nullptr; // we take ownership
o.viable_size = 0;
return *this;
}
void swap(padded_string &o) {
size_t tmp_viable_size = viable_size;
char *tmp_data_ptr = data_ptr;
viable_size = o.viable_size;
data_ptr = o.data_ptr;
o.data_ptr = tmp_data_ptr;
o.viable_size = tmp_viable_size;
}
~padded_string() {
aligned_free_char(data_ptr);
}
size_t size() const { return viable_size; }
size_t length() const { return viable_size; }
char *data() const { return data_ptr; }
private:
padded_string &operator=(const padded_string &o) = delete;
padded_string(const padded_string &o) = delete;
size_t viable_size;
char *data_ptr{nullptr};
}; // padded_string
} // namespace simdjson
#endif
/* end file include/simdjson/padded_string.h */
/* begin file include/simdjson/jsonioutil.h */
#ifndef SIMDJSON_JSONIOUTIL_H
#define SIMDJSON_JSONIOUTIL_H
#include <exception>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
namespace simdjson {
// load a file in memory...
// get a corpus; pad out to cache line so we can always use SIMD
// throws exceptions in case of failure
// first element of the pair is a string (null terminated)
// whereas the second element is the length.
// caller is responsible to free (aligned_free((void*)result.data())))
//
// throws an exception if the file cannot be opened, use try/catch
// try {
// p = get_corpus(filename);
// } catch (const std::exception& e) {
// aligned_free((void*)p.data());
// std::cout << "Could not load the file " << filename << std::endl;
// }
padded_string get_corpus(const std::string &filename);
} // namespace simdjson
#endif
/* end file include/simdjson/jsonioutil.h */
/* begin file include/simdjson/jsonminifier.h */
#ifndef SIMDJSON_JSONMINIFIER_H
#define SIMDJSON_JSONMINIFIER_H
#include <cstddef>
#include <cstdint>
#include <string_view>
namespace simdjson {
// Take input from buf and remove useless whitespace, write it to out; buf and
// out can be the same pointer. Result is null terminated,
// return the string length (minus the null termination).
// The accelerated version of this function only runs on AVX2 hardware.
size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out);
static inline size_t json_minify(const char *buf, size_t len, char *out) {
return json_minify(reinterpret_cast<const uint8_t *>(buf), len,
reinterpret_cast<uint8_t *>(out));
}
static inline size_t json_minify(const std::string_view &p, char *out) {
return json_minify(p.data(), p.size(), out);
}
static inline size_t json_minify(const padded_string &p, char *out) {
return json_minify(p.data(), p.size(), out);
}
} // namespace simdjson
#endif
/* end file include/simdjson/jsonminifier.h */
/* begin file include/simdjson/parsedjson.h */
#ifndef SIMDJSON_PARSEDJSON_H
#define SIMDJSON_PARSEDJSON_H
#include <cstring>
#include <memory>
#define JSON_VALUE_MASK 0xFFFFFFFFFFFFFF
#define DEFAULT_MAX_DEPTH \
1024 // a JSON document with a depth exceeding 1024 is probably de facto
// invalid
namespace simdjson {
/************
* The JSON is parsed to a tape, see the accompanying tape.md file
* for documentation.
***********/
class ParsedJson {
public:
// create a ParsedJson container with zero capacity, call allocate_capacity to
// allocate memory
ParsedJson()=default;
~ParsedJson()=default;
// this is a move only class
ParsedJson(ParsedJson &&p) = default;
ParsedJson(const ParsedJson &p) = delete;
ParsedJson &operator=(ParsedJson &&o) = default;
ParsedJson &operator=(const ParsedJson &o) = delete;
// if needed, allocate memory so that the object is able to process JSON
// documents having up to len bytes and max_depth "depth"
WARN_UNUSED
bool allocate_capacity(size_t len, size_t max_depth = DEFAULT_MAX_DEPTH);
// returns true if the document parsed was valid
bool is_valid() const;
// return an error code corresponding to the last parsing attempt, see
// simdjson.h will return simdjson::UNITIALIZED if no parsing was attempted
int get_error_code() const;
// return the string equivalent of "get_error_code"
std::string get_error_message() const;
// deallocate memory and set capacity to zero, called automatically by the
// destructor
void deallocate();
// this should be called when parsing (right before writing the tapes)
void init();
// print the json to std::ostream (should be valid)
// return false if the tape is likely wrong (e.g., you did not parse a valid
// JSON).
WARN_UNUSED
bool print_json(std::ostream &os) const;
WARN_UNUSED
bool dump_raw_tape(std::ostream &os) const;
// all nodes are stored on the tape using a 64-bit word.
//
// strings, double and ints are stored as
// a 64-bit word with a pointer to the actual value
//
//
//
// for objects or arrays, store [ or { at the beginning and } and ] at the
// end. For the openings ([ or {), we annotate them with a reference to the
// location on the tape of the end, and for then closings (} and ]), we
// annotate them with a reference to the location of the opening
//
//
// this should be considered a private function
really_inline void write_tape(uint64_t val, uint8_t c) {
tape[current_loc++] = val | ((static_cast<uint64_t>(c)) << 56);
}
really_inline void write_tape_s64(int64_t i) {
write_tape(0, 'l');
std::memcpy(&tape[current_loc], &i, sizeof(i));
++current_loc;
}
really_inline void write_tape_u64(uint64_t i) {
write_tape(0, 'u');
tape[current_loc++] = i;
}
really_inline void write_tape_double(double d) {
write_tape(0, 'd');
static_assert(sizeof(d) == sizeof(tape[current_loc]), "mismatch size");
memcpy(&tape[current_loc++], &d, sizeof(double));
// tape[current_loc++] = *((uint64_t *)&d);
}
really_inline uint32_t get_current_loc() const { return current_loc; }
really_inline void annotate_previous_loc(uint32_t saved_loc, uint64_t val) {
tape[saved_loc] |= val;
}
struct InvalidJSON : public std::exception {
const char *what() const noexcept { return "JSON document is invalid"; }
};
template <size_t max_depth> class BasicIterator;
using Iterator = BasicIterator<DEFAULT_MAX_DEPTH>;
size_t byte_capacity{0}; // indicates how many bits are meant to be supported
size_t depth_capacity{0}; // how deep we can go
size_t tape_capacity{0};
size_t string_capacity{0};
uint32_t current_loc{0};
uint32_t n_structural_indexes{0};
std::unique_ptr<uint32_t[]> structural_indexes;
std::unique_ptr<uint64_t[]> tape;
std::unique_ptr<uint32_t[]> containing_scope_offset;
#ifdef SIMDJSON_USE_COMPUTED_GOTO
std::unique_ptr<void*[]> ret_address;
#else
std::unique_ptr<char[]> ret_address;
#endif
std::unique_ptr<uint8_t[]> string_buf;// should be at least byte_capacity
uint8_t *current_string_buf_loc;
bool valid{false};
int error_code{simdjson::UNINITIALIZED};
};
} // namespace simdjson
#endif
/* end file include/simdjson/parsedjson.h */
/* begin file include/simdjson/parsedjsoniterator.h */
#ifndef SIMDJSON_PARSEDJSONITERATOR_H
#define SIMDJSON_PARSEDJSONITERATOR_H
#include <cstring>
#include <iostream>
#include <iterator>
#include <limits>
#include <stdexcept>
namespace simdjson {
template <size_t max_depth> class ParsedJson::BasicIterator {
// might throw InvalidJSON if ParsedJson is invalid
public:
explicit BasicIterator(ParsedJson &pj_);
BasicIterator(const BasicIterator &o) noexcept;
BasicIterator &operator=(const BasicIterator &o) noexcept;
inline bool is_ok() const;
// useful for debuging purposes
inline size_t get_tape_location() const;
// useful for debuging purposes
inline size_t get_tape_length() const;
// returns the current depth (start at 1 with 0 reserved for the fictitious
// root node)
inline size_t get_depth() const;
// A scope is a series of nodes at the same depth, typically it is either an
// object ({) or an array ([). The root node has type 'r'.
inline uint8_t get_scope_type() const;
// move forward in document order
inline bool move_forward();
// retrieve the character code of what we're looking at:
// [{"slutfn are the possibilities
inline uint8_t get_type() const {
return current_type; // short functions should be inlined!
}
// get the int64_t value at this node; valid only if get_type is "l"
inline int64_t get_integer() const {
if (location + 1 >= tape_length) {
return 0; // default value in case of error
}
return static_cast<int64_t>(pj->tape[location + 1]);
}
// get the value as uint64; valid only if if get_type is "u"
inline uint64_t get_unsigned_integer() const {
if (location + 1 >= tape_length) {
return 0; // default value in case of error
}
return pj->tape[location + 1];
}
// get the string value at this node (NULL ended); valid only if get_type is "
// note that tabs, and line endings are escaped in the returned value (see
// print_with_escapes) return value is valid UTF-8, it may contain NULL chars
// within the string: get_string_length determines the true string length.
inline const char *get_string() const {
return reinterpret_cast<const char *>(
pj->string_buf.get() + (current_val & JSON_VALUE_MASK) + sizeof(uint32_t));
}
// return the length of the string in bytes
inline uint32_t get_string_length() const {
uint32_t answer;
memcpy(&answer,
reinterpret_cast<const char *>(pj->string_buf.get() +
(current_val & JSON_VALUE_MASK)),
sizeof(uint32_t));
return answer;
}
// get the double value at this node; valid only if
// get_type() is "d"
inline double get_double() const {
if (location + 1 >= tape_length) {
return std::numeric_limits<double>::quiet_NaN(); // default value in
// case of error
}
double answer;
memcpy(&answer, &pj->tape[location + 1], sizeof(answer));
return answer;
}
inline bool is_object_or_array() const { return is_object() || is_array(); }
inline bool is_object() const { return get_type() == '{'; }
inline bool is_array() const { return get_type() == '['; }
inline bool is_string() const { return get_type() == '"'; }
// Returns true if the current type of node is an signed integer.
// You can get its value with `get_integer()`.
inline bool is_integer() const { return get_type() == 'l'; }
// Returns true if the current type of node is an unsigned integer.
// You can get its value with `get_unsigned_integer()`.
//
// NOTE:
// Only a large value, which is out of range of a 64-bit signed integer, is
// represented internally as an unsigned node. On the other hand, a typical
// positive integer, such as 1, 42, or 1000000, is as a signed node.
// Be aware this function returns false for a signed node.
inline bool is_unsigned_integer() const { return get_type() == 'u'; }
inline bool is_double() const { return get_type() == 'd'; }
inline bool is_number() const {
return is_integer() || is_unsigned_integer() || is_double();
}
inline bool is_true() const { return get_type() == 't'; }
inline bool is_false() const { return get_type() == 'f'; }
inline bool is_null() const { return get_type() == 'n'; }
static bool is_object_or_array(uint8_t type) {
return ((type == '[') || (type == '{'));
}
// when at {, go one level deep, looking for a given key
// if successful, we are left pointing at the value,
// if not, we are still pointing at the object ({)
// (in case of repeated keys, this only finds the first one).
// We seek the key using C's strcmp so if your JSON strings contain
// NULL chars, this would trigger a false positive: if you expect that
// to be the case, take extra precautions.
// Furthermore, we do the comparison character-by-character
// without taking into account Unicode equivalence.
inline bool move_to_key(const char *key);
// as above, but case insensitive lookup (strcmpi instead of strcmp)
inline bool move_to_key_insensitive(const char *key);
// when at {, go one level deep, looking for a given key
// if successful, we are left pointing at the value,
// if not, we are still pointing at the object ({)
// (in case of repeated keys, this only finds the first one).
// The string we search for can contain NULL values.
// Furthermore, we do the comparison character-by-character
// without taking into account Unicode equivalence.
inline bool move_to_key(const char *key, uint32_t length);
// when at a key location within an object, this moves to the accompanying
// value (located next to it). This is equivalent but much faster than
// calling "next()".
inline void move_to_value();
// when at [, go one level deep, and advance to the given index.
// if successful, we are left pointing at the value,
// if not, we are still pointing at the array ([)
inline bool move_to_index(uint32_t index);
// Moves the iterator to the value correspoding to the json pointer.
// Always search from the root of the document.
// if successful, we are left pointing at the value,
// if not, we are still pointing the same value we were pointing before the
// call. The json pointer follows the rfc6901 standard's syntax:
// https://tools.ietf.org/html/rfc6901 However, the standard says "If a
// referenced member name is not unique in an object, the member that is
// referenced is undefined, and evaluation fails". Here we just return the
// first corresponding value. The length parameter is the length of the
// jsonpointer string ('pointer').
bool move_to(const char *pointer, uint32_t length);
// Moves the iterator to the value correspoding to the json pointer.
// Always search from the root of the document.
// if successful, we are left pointing at the value,
// if not, we are still pointing the same value we were pointing before the
// call. The json pointer implementation follows the rfc6901 standard's
// syntax: https://tools.ietf.org/html/rfc6901 However, the standard says
// "If a referenced member name is not unique in an object, the member that
// is referenced is undefined, and evaluation fails". Here we just return
// the first corresponding value.
inline bool move_to(const std::string &pointer) {
return move_to(pointer.c_str(), pointer.length());
}
private:
// Almost the same as move_to(), except it searchs from the current
// position. The pointer's syntax is identical, though that case is not
// handled by the rfc6901 standard. The '/' is still required at the
// beginning. However, contrary to move_to(), the URI Fragment Identifier
// Representation is not supported here. Also, in case of failure, we are
// left pointing at the closest value it could reach. For these reasons it
// is private. It exists because it is used by move_to().
bool relative_move_to(const char *pointer, uint32_t length);
public:
// throughout return true if we can do the navigation, false
// otherwise
// Withing a given scope (series of nodes at the same depth within either an
// array or an object), we move forward.
// Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, {
// and [. At the object ({) or at the array ([), you can issue a "down" to
// visit their content. valid if we're not at the end of a scope (returns
// true).
inline bool next();
// Within a given scope (series of nodes at the same depth within either an
// array or an object), we move backward.
// Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true
// when starting at the end of the scope. At the object ({) or at the array
// ([), you can issue a "down" to visit their content.
// Performance warning: This function is implemented by starting again
// from the beginning of the scope and scanning forward. You should expect
// it to be relatively slow.
inline bool prev();
// Moves back to either the containing array or object (type { or [) from
// within a contained scope.
// Valid unless we are at the first level of the document
inline bool up();
// Valid if we're at a [ or { and it starts a non-empty scope; moves us to
// start of that deeper scope if it not empty. Thus, given [true, null,
// {"a":1}, [1,2]], if we are at the { node, we would move to the "a" node.
inline bool down();
// move us to the start of our current scope,
// a scope is a series of nodes at the same level
inline void to_start_scope();
inline void rewind() {
while (up())
;
}
// void to_end_scope(); // move us to
// the start of our current scope; always succeeds
// print the node we are currently pointing at
bool print(std::ostream &os, bool escape_strings = true) const;
typedef struct {
size_t start_of_scope;
uint8_t scope_type;
} scopeindex_t;
private:
ParsedJson *pj;
size_t depth;
size_t location; // our current location on a tape
size_t tape_length;
uint8_t current_type;
uint64_t current_val;
scopeindex_t depth_index[max_depth];
};
template <size_t max_depth>
WARN_UNUSED bool ParsedJson::BasicIterator<max_depth>::is_ok() const {
return location < tape_length;
}
// useful for debuging purposes
template <size_t max_depth>
size_t ParsedJson::BasicIterator<max_depth>::get_tape_location() const {
return location;
}
// useful for debuging purposes
template <size_t max_depth>
size_t ParsedJson::BasicIterator<max_depth>::get_tape_length() const {
return tape_length;
}
// returns the current depth (start at 1 with 0 reserved for the fictitious root
// node)
template <size_t max_depth>
size_t ParsedJson::BasicIterator<max_depth>::get_depth() const {
return depth;
}
// A scope is a series of nodes at the same depth, typically it is either an
// object ({) or an array ([). The root node has type 'r'.
template <size_t max_depth>
uint8_t ParsedJson::BasicIterator<max_depth>::get_scope_type() const {
return depth_index[depth].scope_type;
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::move_forward() {
if (location + 1 >= tape_length) {
return false; // we are at the end!
}
if ((current_type == '[') || (current_type == '{')) {
// We are entering a new scope
depth++;
assert(depth < max_depth);
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
} else if ((current_type == ']') || (current_type == '}')) {
// Leaving a scope.
depth--;
} else if (is_number()) {
// these types use 2 locations on the tape, not just one.
location += 1;
}
location += 1;
current_val = pj->tape[location];
current_type = (current_val >> 56);
return true;
}
template <size_t max_depth>
void ParsedJson::BasicIterator<max_depth>::move_to_value() {
// assume that we are on a key, so move by 1.
location += 1;
current_val = pj->tape[location];
current_type = (current_val >> 56);
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::move_to_key(const char *key) {
if (down()) {
do {
const bool right_key = (strcmp(get_string(), key) == 0);
move_to_value();
if (right_key) {
return true;
}
} while (next());
up();
}
return false;
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::move_to_key_insensitive(
const char *key) {
if (down()) {
do {
const bool right_key = (simdjson_strcasecmp(get_string(), key) == 0);
move_to_value();
if (right_key) {
return true;
}
} while (next());
up();
}
return false;
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::move_to_key(const char *key,
uint32_t length) {
if (down()) {
do {
bool right_key = ((get_string_length() == length) &&
(memcmp(get_string(), key, length) == 0));
move_to_value();
if (right_key) {
return true;
}
} while (next());
up();
}
return false;
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::move_to_index(uint32_t index) {
if (down()) {
uint32_t i = 0;
for (; i < index; i++) {
if (!next()) {
break;
}
}
if (i == index) {
return true;
}
up();
}
return false;
}
template <size_t max_depth> bool ParsedJson::BasicIterator<max_depth>::prev() {
size_t target_location = location;
to_start_scope();
size_t npos = location;
if (target_location == npos) {
return false; // we were already at the start
}
size_t oldnpos;
// we have that npos < target_location here
do {
oldnpos = npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = (current_val & JSON_VALUE_MASK);
} else {
npos = npos + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
}
} while (npos < target_location);
location = oldnpos;
current_val = pj->tape[location];
current_type = current_val >> 56;
return true;
}
template <size_t max_depth> bool ParsedJson::BasicIterator<max_depth>::up() {
if (depth == 1) {
return false; // don't allow moving back to root
}
to_start_scope();
// next we just move to the previous value
depth--;
location -= 1;
current_val = pj->tape[location];
current_type = (current_val >> 56);
return true;
}
template <size_t max_depth> bool ParsedJson::BasicIterator<max_depth>::down() {
if (location + 1 >= tape_length) {
return false;
}
if ((current_type == '[') || (current_type == '{')) {
size_t npos = (current_val & JSON_VALUE_MASK);
if (npos == location + 2) {
return false; // we have an empty scope
}
depth++;
assert(depth < max_depth);
location = location + 1;
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
current_val = pj->tape[location];
current_type = (current_val >> 56);
return true;
}
return false;
}
template <size_t max_depth>
void ParsedJson::BasicIterator<max_depth>::to_start_scope() {
location = depth_index[depth].start_of_scope;
current_val = pj->tape[location];
current_type = (current_val >> 56);
}
template <size_t max_depth> bool ParsedJson::BasicIterator<max_depth>::next() {
size_t npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = (current_val & JSON_VALUE_MASK);
} else {
npos = location + (is_number() ? 2 : 1);
}
uint64_t next_val = pj->tape[npos];
uint8_t next_type = (next_val >> 56);
if ((next_type == ']') || (next_type == '}')) {
return false; // we reached the end of the scope
}
location = npos;
current_val = next_val;
current_type = next_type;
return true;
}
template <size_t max_depth>
ParsedJson::BasicIterator<max_depth>::BasicIterator(ParsedJson &pj_)
: pj(&pj_), depth(0), location(0), tape_length(0) {
if (!pj->is_valid()) {
throw InvalidJSON();
}
depth_index[0].start_of_scope = location;
current_val = pj->tape[location++];
current_type = (current_val >> 56);
depth_index[0].scope_type = current_type;
if (current_type == 'r') {
tape_length = current_val & JSON_VALUE_MASK;
if (location < tape_length) {
// If we make it here, then depth_capacity must >=2, but the compiler
// may not know this.
current_val = pj->tape[location];
current_type = (current_val >> 56);
depth++;
assert(depth < max_depth);
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
}
} else {
// should never happen
throw InvalidJSON();
}
}
template <size_t max_depth>
ParsedJson::BasicIterator<max_depth>::BasicIterator(
const BasicIterator &o) noexcept
: pj(o.pj), depth(o.depth), location(o.location),
tape_length(o.tape_length), current_type(o.current_type),
current_val(o.current_val) {
memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0]));
}
template <size_t max_depth>
ParsedJson::BasicIterator<max_depth> &ParsedJson::BasicIterator<max_depth>::
operator=(const BasicIterator &o) noexcept {
pj = o.pj;
depth = o.depth;
location = o.location;
tape_length = o.tape_length;
current_type = o.current_type;
current_val = o.current_val;
memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0]));
return *this;
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::print(std::ostream &os,
bool escape_strings) const {
if (!is_ok()) {
return false;
}
switch (current_type) {
case '"': // we have a string
os << '"';
if (escape_strings) {
print_with_escapes(get_string(), os, get_string_length());
} else {
// was: os << get_string();, but given that we can include null chars, we
// have to do something crazier:
std::copy(get_string(), get_string() + get_string_length(),
std::ostream_iterator<char>(os));
}
os << '"';
break;
case 'l': // we have a long int
os << get_integer();
break;
case 'u':
os << get_unsigned_integer();
break;
case 'd':
os << get_double();
break;
case 'n': // we have a null
os << "null";
break;
case 't': // we have a true
os << "true";
break;
case 'f': // we have a false
os << "false";
break;
case '{': // we have an object
case '}': // we end an object
case '[': // we start an array
case ']': // we end an array
os << static_cast<char>(current_type);
break;
default:
return false;
}
return true;
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::move_to(const char *pointer,
uint32_t length) {
char *new_pointer = nullptr;
if (pointer[0] == '#') {
// Converting fragment representation to string representation
new_pointer = new char[length];
uint32_t new_length = 0;
for (uint32_t i = 1; i < length; i++) {
if (pointer[i] == '%' && pointer[i + 1] == 'x') {
try {
int fragment =
std::stoi(std::string(&pointer[i + 2], 2), nullptr, 16);
if (fragment == '\\' || fragment == '"' || (fragment <= 0x1F)) {
// escaping the character
new_pointer[new_length] = '\\';
new_length++;
}
new_pointer[new_length] = fragment;
i += 3;
} catch (std::invalid_argument &) {
delete[] new_pointer;
return false; // the fragment is invalid
}
} else {
new_pointer[new_length] = pointer[i];
}
new_length++;
}
length = new_length;
pointer = new_pointer;
}
// saving the current state
size_t depth_s = depth;
size_t location_s = location;
uint8_t current_type_s = current_type;
uint64_t current_val_s = current_val;
rewind(); // The json pointer is used from the root of the document.
bool found = relative_move_to(pointer, length);
delete[] new_pointer;
if (!found) {
// since the pointer has found nothing, we get back to the original
// position.
depth = depth_s;
location = location_s;
current_type = current_type_s;
current_val = current_val_s;
}
return found;
}
template <size_t max_depth>
bool ParsedJson::BasicIterator<max_depth>::relative_move_to(const char *pointer,
uint32_t length) {
if (length == 0) {
// returns the whole document
return true;
}
if (pointer[0] != '/') {
// '/' must be the first character
return false;
}
// finding the key in an object or the index in an array
std::string key_or_index;
uint32_t offset = 1;
// checking for the "-" case
if (is_array() && pointer[1] == '-') {
if (length != 2) {
// the pointer must be exactly "/-"
// there can't be anything more after '-' as an index
return false;
}
key_or_index = '-';
offset = length; // will skip the loop coming right after
}
// We either transform the first reference token to a valid json key
// or we make sure it is a valid index in an array.
for (; offset < length; offset++) {
if (pointer[offset] == '/') {
// beginning of the next key or index
break;
}
if (is_array() && (pointer[offset] < '0' || pointer[offset] > '9')) {
// the index of an array must be an integer
// we also make sure std::stoi won't discard whitespaces later
return false;
}
if (pointer[offset] == '~') {
// "~1" represents "/"
if (pointer[offset + 1] == '1') {
key_or_index += '/';
offset++;
continue;
}
// "~0" represents "~"
if (pointer[offset + 1] == '0') {
key_or_index += '~';
offset++;
continue;
}
}
if (pointer[offset] == '\\') {
if (pointer[offset + 1] == '\\' || pointer[offset + 1] == '"' ||
(pointer[offset + 1] <= 0x1F)) {
key_or_index += pointer[offset + 1];
offset++;
continue;
}
return false; // invalid escaped character
}
if (pointer[offset] == '\"') {
// unescaped quote character. this is an invalid case.
// lets do nothing and assume most pointers will be valid.
// it won't find any corresponding json key anyway.
// return false;
}
key_or_index += pointer[offset];
}
bool found = false;
if (is_object()) {
if (move_to_key(key_or_index.c_str(), key_or_index.length())) {
found = relative_move_to(pointer + offset, length - offset);
}
} else if (is_array()) {
if (key_or_index == "-") { // handling "-" case first
if (down()) {
while (next())
; // moving to the end of the array
// moving to the nonexistent value right after...
size_t npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = (current_val & JSON_VALUE_MASK);
} else {
npos =
location + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
}
location = npos;
current_val = pj->tape[npos];
current_type = (current_val >> 56);
return true; // how could it fail ?
}
} else { // regular numeric index
// The index can't have a leading '0'
if (key_or_index[0] == '0' && key_or_index.length() > 1) {
return false;
}
// it cannot be empty
if (key_or_index.length() == 0) {
return false;
}
// we already checked the index contains only valid digits
uint32_t index = std::stoi(key_or_index);
if (move_to_index(index)) {
found = relative_move_to(pointer + offset, length - offset);
}
}
}
return found;
}
} // namespace simdjson
#endif
/* end file include/simdjson/parsedjsoniterator.h */
/* begin file include/simdjson/stage1_find_marks.h */
#ifndef SIMDJSON_STAGE1_FIND_MARKS_H
#define SIMDJSON_STAGE1_FIND_MARKS_H
namespace simdjson {
// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings.
// The caller should still ensure that the input is valid UTF-8. If you are processing substrings,
// you may want to call on a function like trimmed_length_safe_utf8.
// A function like find_last_json_buf_idx may also prove useful.
template <Architecture T = Architecture::NATIVE>
int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming);
// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings.
// The caller should still ensure that the input is valid UTF-8. If you are processing substrings,
// you may want to call on a function like trimmed_length_safe_utf8.
// A function like find_last_json_buf_idx may also prove useful.
template <Architecture T = Architecture::NATIVE>
int find_structural_bits(const char *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) {
return find_structural_bits<T>((const uint8_t *)buf, len, pj, streaming);
}
template <Architecture T = Architecture::NATIVE>
int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) {
return find_structural_bits<T>(buf, len, pj, false);
}
template <Architecture T = Architecture::NATIVE>
int find_structural_bits(const char *buf, size_t len, simdjson::ParsedJson &pj) {
return find_structural_bits<T>((const uint8_t *)buf, len, pj);
}
} // namespace simdjson
#endif
/* end file include/simdjson/stage1_find_marks.h */
/* begin file include/simdjson/stage2_build_tape.h */
#ifndef SIMDJSON_STAGE2_BUILD_TAPE_H
#define SIMDJSON_STAGE2_BUILD_TAPE_H
namespace simdjson {
void init_state_machine();
template <Architecture T = Architecture::NATIVE>
WARN_UNUSED int
unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);
template <Architecture T = Architecture::NATIVE>
WARN_UNUSED int
unified_machine(const char *buf, size_t len, ParsedJson &pj) {
return unified_machine<T>(reinterpret_cast<const uint8_t *>(buf), len, pj);
}
// Streaming
template <Architecture T = Architecture::NATIVE>
WARN_UNUSED int
unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj, size_t &next_json);
template <Architecture T = Architecture::NATIVE>
int unified_machine(const char *buf, size_t len, ParsedJson &pj, size_t &next_json) {
return unified_machine<T>(reinterpret_cast<const uint8_t *>(buf), len, pj, next_json);
}
} // namespace simdjson
#endif
/* end file include/simdjson/stage2_build_tape.h */
/* begin file include/simdjson/jsonparser.h */
#ifndef SIMDJSON_JSONPARSER_H
#define SIMDJSON_JSONPARSER_H
#include <string>
namespace simdjson {
// json_parse_implementation is the generic function, it is specialized for
// various architectures, e.g., as
// json_parse_implementation<Architecture::HASWELL> or
// json_parse_implementation<Architecture::ARM64>
template <Architecture T>
int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj,
bool realloc_if_needed = true) {
if (pj.byte_capacity < len) {
return simdjson::CAPACITY;
}
bool reallocated = false;
if (realloc_if_needed) {
const uint8_t *tmp_buf = buf;
buf = (uint8_t *)allocate_padded_buffer(len);
if (buf == NULL)
return simdjson::MEMALLOC;
memcpy((void *)buf, tmp_buf, len);
reallocated = true;
} // if(realloc_if_needed) {
int stage1_is_ok = simdjson::find_structural_bits<T>(buf, len, pj);
if (stage1_is_ok != simdjson::SUCCESS) {
if (reallocated) { // must free before we exit
aligned_free((void *)buf);
}
pj.error_code = stage1_is_ok;
return pj.error_code;
}
int res = unified_machine<T>(buf, len, pj);
if (reallocated) {
aligned_free((void *)buf);
}
return res;
}
// Parse a document found in buf.
//
// The content should be a valid JSON document encoded as UTF-8. If there is a
// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
// discouraged.
//
// You need to preallocate ParsedJson with a capacity of len (e.g.,
// pj.allocate_capacity(len)).
//
// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
// or an error code from simdjson/simdjson.h in case of failure such as
// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
// the simdjson::error_message function converts these error codes into a
// string).
//
// You can also check validity by calling pj.is_valid(). The same ParsedJson can
// be reused for other documents.
//
// If realloc_if_needed is true (default) then a temporary buffer is created
// when needed during processing (a copy of the input string is made). The input
// buf should be readable up to buf + len + SIMDJSON_PADDING if
// realloc_if_needed is false, all bytes at and after buf + len are ignored
// (can be garbage). The ParsedJson object can be reused.
int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj,
bool realloc_if_needed = true);
// Parse a document found in buf.
//
// The content should be a valid JSON document encoded as UTF-8. If there is a
// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
// discouraged.
//
// You need to preallocate ParsedJson with a capacity of len (e.g.,
// pj.allocate_capacity(len)).
//
// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
// or an error code from simdjson/simdjson.h in case of failure such as
// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
// the simdjson::error_message function converts these error codes into a
// string).
//
// You can also check validity
// by calling pj.is_valid(). The same ParsedJson can be reused for other
// documents.
//
// If realloc_if_needed is true (default) then a temporary buffer is created
// when needed during processing (a copy of the input string is made). The input
// buf should be readable up to buf + len + SIMDJSON_PADDING if
// realloc_if_needed is false, all bytes at and after buf + len are ignored
// (can be garbage). The ParsedJson object can be reused.
int json_parse(const char *buf, size_t len, ParsedJson &pj,
bool realloc_if_needed = true);
// We do not want to allow implicit conversion from C string to std::string.
int json_parse(const char *buf, ParsedJson &pj) = delete;
// Parse a document found in in string s.
// You need to preallocate ParsedJson with a capacity of len (e.g.,
// pj.allocate_capacity(len)).
//
// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
// or an error code from simdjson/simdjson.h in case of failure such as
// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
// the simdjson::error_message function converts these error codes into a
// string).
//
// A temporary buffer is created when needed during processing
// (a copy of the input string is made).
inline int json_parse(const std::string &s, ParsedJson &pj) {
return json_parse(s.data(), s.length(), pj, true);
}
// Parse a document found in in string s.
//
// The content should be a valid JSON document encoded as UTF-8. If there is a
// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
// discouraged.
//
// You need to preallocate ParsedJson with a capacity of len (e.g.,
// pj.allocate_capacity(len)).
//
// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
// or an error code from simdjson/simdjson.h in case of failure such as
// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
// the simdjson::error_message function converts these error codes into a
// string).
//
// You can also check validity
// by calling pj.is_valid(). The same ParsedJson can be reused for other
// documents.
inline int json_parse(const padded_string &s, ParsedJson &pj) {
return json_parse(s.data(), s.length(), pj, false);
}
// Build a ParsedJson object. You can check validity
// by calling pj.is_valid(). This does the memory allocation needed for
// ParsedJson. If realloc_if_needed is true (default) then a temporary buffer is
// created when needed during processing (a copy of the input string is made).
//
// The input buf should be readable up to buf + len + SIMDJSON_PADDING if
// realloc_if_needed is false, all bytes at and after buf + len are ignored
// (can be garbage).
//
// The content should be a valid JSON document encoded as UTF-8. If there is a
// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
// discouraged.
//
// This is a convenience function which calls json_parse.
WARN_UNUSED
ParsedJson build_parsed_json(const uint8_t *buf, size_t len,
bool realloc_if_needed = true);
WARN_UNUSED
// Build a ParsedJson object. You can check validity
// by calling pj.is_valid(). This does the memory allocation needed for
// ParsedJson. If realloc_if_needed is true (default) then a temporary buffer is
// created when needed during processing (a copy of the input string is made).
//
// The input buf should be readable up to buf + len + SIMDJSON_PADDING if
// realloc_if_needed is false, all bytes at and after buf + len are ignored
// (can be garbage).
//
//
// The content should be a valid JSON document encoded as UTF-8. If there is a
// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
// discouraged.
//
// This is a convenience function which calls json_parse.
inline ParsedJson build_parsed_json(const char *buf, size_t len,
bool realloc_if_needed = true) {
return build_parsed_json(reinterpret_cast<const uint8_t *>(buf), len,
realloc_if_needed);
}
// We do not want to allow implicit conversion from C string to std::string.
ParsedJson build_parsed_json(const char *buf) = delete;
// Parse a document found in in string s.
// You need to preallocate ParsedJson with a capacity of len (e.g.,
// pj.allocate_capacity(len)). Return SUCCESS (an integer = 0) in case of a
// success. You can also check validity by calling pj.is_valid(). The same
// ParsedJson can be reused for other documents.
//
// A temporary buffer is created when needed during processing
// (a copy of the input string is made).
//
// The content should be a valid JSON document encoded as UTF-8. If there is a
// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
// discouraged.
//
// This is a convenience function which calls json_parse.
WARN_UNUSED
inline ParsedJson build_parsed_json(const std::string &s) {
return build_parsed_json(s.data(), s.length(), true);
}
// Parse a document found in in string s.
// You need to preallocate ParsedJson with a capacity of len (e.g.,
// pj.allocate_capacity(len)). Return SUCCESS (an integer = 0) in case of a
// success. You can also check validity by calling pj.is_valid(). The same
// ParsedJson can be reused for other documents.
//
// The content should be a valid JSON document encoded as UTF-8. If there is a
// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
// discouraged.
//
// This is a convenience function which calls json_parse.
WARN_UNUSED
inline ParsedJson build_parsed_json(const padded_string &s) {
return build_parsed_json(s.data(), s.length(), false);
}
} // namespace simdjson
#endif
/* end file include/simdjson/jsonparser.h */
/* begin file include/simdjson/jsonstream.h */
#ifndef SIMDJSON_JSONSTREAM_H
#define SIMDJSON_JSONSTREAM_H
#include <algorithm>
#include <limits>
#include <stdexcept>
#include <thread>
/* begin file src/jsoncharutils.h */
#ifndef SIMDJSON_JSONCHARUTILS_H
#define SIMDJSON_JSONCHARUTILS_H
namespace simdjson {
// structural chars here are
// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL)
// we are also interested in the four whitespace characters
// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
// these are the chars that can follow a true/false/null or number atom
// and nothing else
const uint32_t structural_or_whitespace_or_null_negated[256] = {
0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
// return non-zero if not a structural or whitespace char
// zero otherwise
really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) {
return structural_or_whitespace_or_null_negated[c];
}
const uint32_t structural_or_whitespace_negated[256] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
// return non-zero if not a structural or whitespace char
// zero otherwise
really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) {
return structural_or_whitespace_negated[c];
}
const uint32_t structural_or_whitespace_or_null[256] = {
1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) {
return structural_or_whitespace_or_null[c];
}
const uint32_t structural_or_whitespace[256] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
really_inline uint32_t is_structural_or_whitespace(uint8_t c) {
return structural_or_whitespace[c];
}
const uint32_t digit_to_val32[886] = {
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0x0, 0x1, 0x2, 0x3, 0x4, 0x5,
0x6, 0x7, 0x8, 0x9, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa,
0xb, 0xc, 0xd, 0xe, 0xf, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xa, 0xb, 0xc, 0xd, 0xe,
0xf, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0x0, 0x10, 0x20, 0x30, 0x40, 0x50,
0x60, 0x70, 0x80, 0x90, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa0,
0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0,
0xf0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0x0, 0x100, 0x200, 0x300, 0x400, 0x500,
0x600, 0x700, 0x800, 0x900, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa00,
0xb00, 0xc00, 0xd00, 0xe00, 0xf00, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xa00, 0xb00, 0xc00, 0xd00, 0xe00,
0xf00, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0x0, 0x1000, 0x2000, 0x3000, 0x4000, 0x5000,
0x6000, 0x7000, 0x8000, 0x9000, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa000,
0xb000, 0xc000, 0xd000, 0xe000, 0xf000, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xa000, 0xb000, 0xc000, 0xd000, 0xe000,
0xf000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF};
// returns a value with the high 16 bits set if not valid
// otherwise returns the conversion of the 4 hex digits at src into the bottom
// 16 bits of the 32-bit return register
//
// see
// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/
static inline uint32_t hex_to_u32_nocheck(
const uint8_t *src) { // strictly speaking, static inline is a C-ism
uint32_t v1 = digit_to_val32[630 + src[0]];
uint32_t v2 = digit_to_val32[420 + src[1]];
uint32_t v3 = digit_to_val32[210 + src[2]];
uint32_t v4 = digit_to_val32[0 + src[3]];
return v1 | v2 | v3 | v4;
}
// returns true if the provided byte value is a
// "continuing" UTF-8 value, that is, if it starts with
// 0b10...
static inline bool is_utf8_continuing(char c) {
// in 2 complement's notation, values start at 0b10000 (-128)... and
// go up to 0b11111 (-1)... so we want all values from -128 to -65 (which is 0b10111111)
return ((signed char)c) <= -65;
}
// returns true if the provided byte value is an ASCII character
static inline bool is_ascii(char c) {
return ((unsigned char)c) <= 127;
}
// if the string ends with UTF-8 values, backtrack
// up to the first ASCII character. May return 0.
static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) {
while ((len > 0) and (not is_ascii(c[len - 1]))) {
len--;
}
return len;
}
// given a code point cp, writes to c
// the utf-8 code, outputting the length in
// bytes, if the length is zero, the code point
// is invalid
//
// This can possibly be made faster using pdep
// and clz and table lookups, but JSON documents
// have few escaped code points, and the following
// function looks cheap.
//
// Note: we assume that surrogates are treated separately
//
inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
if (cp <= 0x7F) {
c[0] = cp;
return 1; // ascii
}
if (cp <= 0x7FF) {
c[0] = (cp >> 6) + 192;
c[1] = (cp & 63) + 128;
return 2; // universal plane
// Surrogates are treated elsewhere...
//} //else if (0xd800 <= cp && cp <= 0xdfff) {
// return 0; // surrogates // could put assert here
} else if (cp <= 0xFFFF) {
c[0] = (cp >> 12) + 224;
c[1] = ((cp >> 6) & 63) + 128;
c[2] = (cp & 63) + 128;
return 3;
} else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this
// is not needed
c[0] = (cp >> 18) + 240;
c[1] = ((cp >> 12) & 63) + 128;
c[2] = ((cp >> 6) & 63) + 128;
c[3] = (cp & 63) + 128;
return 4;
}
// will return 0 when the code point was too large.
return 0; // bad r
}
} // namespace simdjson
#endif
/* end file src/jsoncharutils.h */
namespace simdjson {
/*************************************************************************************
* The main motivation for this piece of software is to achieve maximum speed
*and offer
* good quality of life while parsing files containing multiple JSON documents.
*
* Since we want to offer flexibility and not restrict ourselves to a specific
*file
* format, we support any file that contains any valid JSON documents separated
*by one
* or more character that is considered a whitespace by the JSON spec.
* Namely: space, nothing, linefeed, carriage return, horizontal tab.
* Anything that is not whitespace will be parsed as a JSON document and could
*lead
* to failure.
*
* To offer maximum parsing speed, our implementation processes the data inside
*the
* buffer by batches and their size is defined by the parameter "batch_size".
* By loading data in batches, we can optimize the time spent allocating data in
*the
* ParsedJson and can also open the possibility of multi-threading.
* The batch_size must be at least as large as the biggest document in the file,
*but
* not too large in order to submerge the chached memory. We found that 1MB is
* somewhat a sweet spot for now. Eventually, this batch_size could be fully
* automated and be optimal at all times.
************************************************************************************/
/**
* The template parameter (string_container) must
* support the data() and size() methods, returning a pointer
* to a char* and to the number of bytes respectively.
* The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end
* of the string, so if you do not use a padded_string container,
* you have the responsability to overallocated. If you fail to
* do so, your software may crash if you cross a page boundary,
* and you should expect memory checkers to object.
* Most users should use a simdjson::padded_string.
*/
template <class string_container = padded_string> class JsonStream {
public:
/* Create a JsonStream object that can be used to parse sequentially the valid
* JSON documents found in the buffer "buf".
*
* The batch_size must be at least as large as the biggest document in the
* file, but
* not too large to submerge the cached memory. We found that 1MB is
* somewhat a sweet spot for now.
*
* The user is expected to call the following json_parse method to parse the
* next
* valid JSON document found in the buffer. This method can and is expected
* to be
* called in a loop.
*
* Various methods are offered to keep track of the status, like
* get_current_buffer_loc,
* get_n_parsed_docs, get_n_bytes_parsed, etc.
*
* */
JsonStream(const string_container &s, size_t batch_size = 1000000);
~JsonStream();
/* Parse the next document found in the buffer previously given to JsonStream.
* The content should be a valid JSON document encoded as UTF-8. If there is a
* UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
* discouraged.
*
* You do NOT need to pre-allocate ParsedJson. This function takes care of
* pre-allocating a capacity defined by the batch_size defined when creating
the
* JsonStream object.
*
* The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in
case
* of success and indicates that the buffer still contains more data to be
parsed,
* meaning this function can be called again to return the next JSON document
* after this one.
*
* The function returns simdjson::SUCCESS (as integer = 0) in case of success
* and indicates that the buffer has successfully been parsed to the end.
* Every document it contained has been parsed without error.
*
* The function returns an error code from simdjson/simdjson.h in case of
failure
* such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and
so forth;
* the simdjson::error_message function converts these error codes into a
* string).
*
* You can also check validity by calling pj.is_valid(). The same ParsedJson
can
* and should be reused for the other documents in the buffer. */
int json_parse(ParsedJson &pj);
/* Returns the location (index) of where the next document should be in the
* buffer.
* Can be used for debugging, it tells the user the position of the end of the
* last
* valid JSON document parsed*/
inline size_t get_current_buffer_loc() const { return current_buffer_loc; }
/* Returns the total amount of complete documents parsed by the JsonStream,
* in the current buffer, at the given time.*/
inline size_t get_n_parsed_docs() const { return n_parsed_docs; }
/* Returns the total amount of data (in bytes) parsed by the JsonStream,
* in the current buffer, at the given time.*/
inline size_t get_n_bytes_parsed() const { return n_bytes_parsed; }
private:
inline const char *buf() const { return str.data() + str_start; }
inline void advance(size_t offset) { str_start += offset; }
inline size_t remaining() const { return str.size() - str_start; }
const string_container &str;
size_t _batch_size; // this is actually variable!
size_t str_start{0};
size_t next_json{0};
bool load_next_batch{true};
size_t current_buffer_loc{0};
#ifdef SIMDJSON_THREADS_ENABLED
size_t last_json_buffer_loc{0};
#endif
size_t n_parsed_docs{0};
size_t n_bytes_parsed{0};
#ifdef SIMDJSON_THREADS_ENABLED
int stage1_is_ok_thread{0};
std::thread stage_1_thread;
simdjson::ParsedJson pj_thread;
#endif
}; // end of class JsonStream
/* This algorithm is used to quickly identify the buffer position of
* the last JSON document inside the current batch.
*
* It does its work by finding the last pair of structural characters
* that represent the end followed by the start of a document.
*
* Simply put, we iterate over the structural characters, starting from
* the end. We consider that we found the end of a JSON document when the
* first element of the pair is NOT one of these characters: '{' '[' ';' ','
* and when the second element is NOT one of these characters: '}' '}' ';' ','.
*
* This simple comparison works most of the time, but it does not cover cases
* where the batch's structural indexes contain a perfect amount of documents.
* In such a case, we do not have access to the structural index which follows
* the last document, therefore, we do not have access to the second element in
* the pair, and means that we cannot identify the last document. To fix this
* issue, we keep a count of the open and closed curly/square braces we found
* while searching for the pair. When we find a pair AND the count of open and
* closed curly/square braces is the same, we know that we just passed a
* complete
* document, therefore the last json buffer location is the end of the batch
* */
inline size_t find_last_json_buf_idx(const char *buf, size_t size,
const ParsedJson &pj) {
// this function can be generally useful
if (pj.n_structural_indexes == 0)
return 0;
auto last_i = pj.n_structural_indexes - 1;
if (pj.structural_indexes[last_i] == size) {
if (last_i == 0)
return 0;
last_i = pj.n_structural_indexes - 2;
}
auto arr_cnt = 0;
auto obj_cnt = 0;
for (auto i = last_i; i > 0; i--) {
auto idxb = pj.structural_indexes[i];
switch (buf[idxb]) {
case ':':
case ',':
continue;
case '}':
obj_cnt--;
continue;
case ']':
arr_cnt--;
continue;
case '{':
obj_cnt++;
break;
case '[':
arr_cnt++;
break;
}
auto idxa = pj.structural_indexes[i - 1];
switch (buf[idxa]) {
case '{':
case '[':
case ':':
case ',':
continue;
}
if (!arr_cnt && !obj_cnt) {
return last_i + 1;
}
return i;
}
return 0;
}
// Everything in the following anonymous namespace should go.
// It is a hack.
namespace {
typedef int (*stage1_functype)(const char *buf, size_t len,
simdjson::ParsedJson &pj, bool streaming);
typedef int (*stage2_functype)(const char *buf, size_t len,
simdjson::ParsedJson &pj, size_t &next_json);
stage1_functype best_stage1;
stage2_functype best_stage2;
//// TODO: generalize this set of functions. We don't want to have a copy in
/// jsonparser.cpp
void find_the_best_supported_implementation() {
uint32_t supports = simdjson::detect_supported_architectures();
// Order from best to worst (within architecture)
#ifdef IS_X86_64
constexpr uint32_t haswell_flags =
simdjson::instruction_set::AVX2 | simdjson::instruction_set::PCLMULQDQ |
simdjson::instruction_set::BMI1 | simdjson::instruction_set::BMI2;
constexpr uint32_t westmere_flags =
simdjson::instruction_set::SSE42 | simdjson::instruction_set::PCLMULQDQ;
if ((haswell_flags & supports) == haswell_flags) {
best_stage1 =
simdjson::find_structural_bits<simdjson::Architecture::HASWELL>;
best_stage2 = simdjson::unified_machine<simdjson::Architecture::HASWELL>;
return;
}
if ((westmere_flags & supports) == westmere_flags) {
best_stage1 =
simdjson::find_structural_bits<simdjson::Architecture::WESTMERE>;
best_stage2 = simdjson::unified_machine<simdjson::Architecture::WESTMERE>;
return;
}
#endif
#ifdef IS_ARM64
if (supports & instruction_set::NEON) {
best_stage1 = simdjson::find_structural_bits<Architecture::ARM64>;
best_stage2 = simdjson::unified_machine<Architecture::ARM64>;
return;
}
#endif
// we throw an exception since this should not be recoverable
throw new std::runtime_error("unsupported architecture");
}
} // anonymous namespace
template <class string_container>
JsonStream<string_container>::JsonStream(const string_container &s,
size_t batchSize)
: str(s), _batch_size(batchSize) {
find_the_best_supported_implementation();
}
template <class string_container> JsonStream<string_container>::~JsonStream() {
#ifdef SIMDJSON_THREADS_ENABLED
if (stage_1_thread.joinable()) {
stage_1_thread.join();
}
#endif
}
#ifdef SIMDJSON_THREADS_ENABLED
// threaded version of json_parse
// todo: simplify this code further
template <class string_container>
int JsonStream<string_container>::json_parse(ParsedJson &pj) {
if (unlikely(pj.byte_capacity == 0)) {
const bool allocok = pj.allocate_capacity(_batch_size);
if (!allocok) {
pj.error_code = simdjson::MEMALLOC;
return pj.error_code;
}
} else if (unlikely(pj.byte_capacity < _batch_size)) {
pj.error_code = simdjson::CAPACITY;
return pj.error_code;
}
if (unlikely(pj_thread.byte_capacity < _batch_size)) {
const bool allocok_thread = pj_thread.allocate_capacity(_batch_size);
if (!allocok_thread) {
pj.error_code = simdjson::MEMALLOC;
return pj.error_code;
}
}
if (unlikely(load_next_batch)) {
// First time loading
if (!stage_1_thread.joinable()) {
_batch_size = (std::min)(_batch_size, remaining());
_batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size);
if (_batch_size == 0) {
pj.error_code = simdjson::UTF8_ERROR;
return pj.error_code;
}
int stage1_is_ok = best_stage1(buf(), _batch_size, pj, true);
if (stage1_is_ok != simdjson::SUCCESS) {
pj.error_code = stage1_is_ok;
return pj.error_code;
}
size_t last_index = find_last_json_buf_idx(buf(), _batch_size, pj);
if (last_index == 0) {
if (pj.n_structural_indexes == 0) {
pj.error_code = simdjson::EMPTY;
return pj.error_code;
}
} else {
pj.n_structural_indexes = last_index + 1;
}
}
// the second thread is running or done.
else {
stage_1_thread.join();
if (stage1_is_ok_thread != simdjson::SUCCESS) {
pj.error_code = stage1_is_ok_thread;
return pj.error_code;
}
std::swap(pj.structural_indexes, pj_thread.structural_indexes);
pj.n_structural_indexes = pj_thread.n_structural_indexes;
advance(last_json_buffer_loc);
n_bytes_parsed += last_json_buffer_loc;
}
// let us decide whether we will start a new thread
if (remaining() - _batch_size > 0) {
last_json_buffer_loc =
pj.structural_indexes[find_last_json_buf_idx(buf(), _batch_size, pj)];
_batch_size = (std::min)(_batch_size, remaining() - last_json_buffer_loc);
if (_batch_size > 0) {
_batch_size = trimmed_length_safe_utf8(
(const char *)(buf() + last_json_buffer_loc), _batch_size);
if (_batch_size == 0) {
pj.error_code = simdjson::UTF8_ERROR;
return pj.error_code;
}
// let us capture read-only variables
const char *const b = buf() + last_json_buffer_loc;
const size_t bs = _batch_size;
// we call the thread on a lambda that will update
// this->stage1_is_ok_thread
// there is only one thread that may write to this value
stage_1_thread = std::thread([this, b, bs] {
this->stage1_is_ok_thread = best_stage1(b, bs, this->pj_thread, true);
});
}
}
next_json = 0;
load_next_batch = false;
} // load_next_batch
int res = best_stage2(buf(), remaining(), pj, next_json);
if (res == simdjson::SUCCESS_AND_HAS_MORE) {
n_parsed_docs++;
current_buffer_loc = pj.structural_indexes[next_json];
load_next_batch = (current_buffer_loc == last_json_buffer_loc);
} else if (res == simdjson::SUCCESS) {
n_parsed_docs++;
if (remaining() > _batch_size) {
current_buffer_loc = pj.structural_indexes[next_json - 1];
load_next_batch = true;
res = simdjson::SUCCESS_AND_HAS_MORE;
}
}
return res;
}
#else // SIMDJSON_THREADS_ENABLED
// single-threaded version of json_parse
template <class string_container>
int JsonStream<string_container>::json_parse(ParsedJson &pj) {
if (unlikely(pj.byte_capacity == 0)) {
const bool allocok = pj.allocate_capacity(_batch_size);
if (!allocok) {
pj.error_code = simdjson::MEMALLOC;
return pj.error_code;
}
} else if (unlikely(pj.byte_capacity < _batch_size)) {
pj.error_code = simdjson::CAPACITY;
return pj.error_code;
}
if (unlikely(load_next_batch)) {
advance(current_buffer_loc);
n_bytes_parsed += current_buffer_loc;
_batch_size = (std::min)(_batch_size, remaining());
_batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size);
int stage1_is_ok = best_stage1(buf(), _batch_size, pj, true);
if (stage1_is_ok != simdjson::SUCCESS) {
pj.error_code = stage1_is_ok;
return pj.error_code;
}
size_t last_index = find_last_json_buf_idx(buf(), _batch_size, pj);
if (last_index == 0) {
if (pj.n_structural_indexes == 0) {
pj.error_code = simdjson::EMPTY;
return pj.error_code;
}
} else {
pj.n_structural_indexes = last_index + 1;
}
load_next_batch = false;
} // load_next_batch
int res = best_stage2(buf(), remaining(), pj, next_json);
if (likely(res == simdjson::SUCCESS_AND_HAS_MORE)) {
n_parsed_docs++;
current_buffer_loc = pj.structural_indexes[next_json];
} else if (res == simdjson::SUCCESS) {
n_parsed_docs++;
if (remaining() > _batch_size) {
current_buffer_loc = pj.structural_indexes[next_json - 1];
next_json = 1;
load_next_batch = true;
res = simdjson::SUCCESS_AND_HAS_MORE;
}
}
return res;
}
#endif // SIMDJSON_THREADS_ENABLED
} // end of namespace simdjson
#endif // SIMDJSON_JSONSTREAM_H
/* end file src/jsoncharutils.h */
Reference in New Issue View Git Blame Copy Permalink