jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity
simdjson/singleheader/simdjson.h

4051 lines
132 KiB
C++
Raw Blame History

/* auto-generated on Thu Mar 5 10:30:07 PST 2020. Do not edit! */
/* begin file include/simdjson.h */
#ifndef SIMDJSON_H
#define SIMDJSON_H
/* begin file include/simdjson/compiler_check.h */
#ifndef SIMDJSON_COMPILER_CHECK_H
#define SIMDJSON_COMPILER_CHECK_H
#ifndef __cplusplus
#error simdjson requires a C++ compiler
#endif
#ifndef SIMDJSON_CPLUSPLUS
#if defined(_MSVC_LANG) && !defined(__clang__)
#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG)
#else
#define SIMDJSON_CPLUSPLUS __cplusplus
#endif
#endif
#if (SIMDJSON_CPLUSPLUS < 201703L)
#error simdjson requires a compiler compliant with the C++17 standard
#endif
#endif // SIMDJSON_COMPILER_CHECK_H
/* end file include/simdjson/compiler_check.h */
// Public API
/* begin file include/simdjson/simdjson_version.h */
// /include/simdjson/simdjson_version.h automatically generated by release.py,
// do not change by hand
#ifndef SIMDJSON_SIMDJSON_VERSION_H
#define SIMDJSON_SIMDJSON_VERSION_H
#define SIMDJSON_VERSION 0.2.1
namespace simdjson {
enum {
SIMDJSON_VERSION_MAJOR = 0,
SIMDJSON_VERSION_MINOR = 2,
SIMDJSON_VERSION_REVISION = 1
};
}
#endif // SIMDJSON_SIMDJSON_VERSION_H
/* end file include/simdjson/simdjson_version.h */
/* begin file include/simdjson/error.h */
#ifndef SIMDJSON_ERROR_H
#define SIMDJSON_ERROR_H
#include <string>
namespace simdjson {
enum error_code {
SUCCESS = 0,
SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data
CAPACITY, // This parser can't support a document that big
MEMALLOC, // Error allocating memory, most likely out of memory
TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this
// is a generic error
DEPTH_ERROR, // Your document exceeds the user-specified depth limitation
STRING_ERROR, // Problem while parsing a string
T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't'
F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f'
N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n'
NUMBER_ERROR, // Problem while parsing a number
UTF8_ERROR, // the input is not valid UTF-8
UNINITIALIZED, // unknown error, or uninitialized document
EMPTY, // no structural element found
UNESCAPED_CHARS, // found unescaped characters in a string.
UNCLOSED_STRING, // missing quote at the end
UNSUPPORTED_ARCHITECTURE, // unsupported architecture
INCORRECT_TYPE, // JSON element has a different type than user expected
NUMBER_OUT_OF_RANGE, // JSON number does not fit in 64 bits
NO_SUCH_FIELD, // JSON field not found in object
UNEXPECTED_ERROR // indicative of a bug in simdjson
};
const std::string &error_message(error_code error) noexcept;
struct invalid_json : public std::exception {
invalid_json(error_code _error) : error{_error} { }
const char *what() const noexcept { return error_message(error); }
error_code error;
};
// TODO these are deprecated, remove
using ErrorValues = error_code;
inline const std::string &error_message(int error) noexcept { return error_message(error_code(error)); }
} // namespace simdjson
#endif // SIMDJSON_ERROR_H
/* end file include/simdjson/error.h */
/* begin file include/simdjson/padded_string.h */
#ifndef SIMDJSON_PADDING_STRING_H
#define SIMDJSON_PADDING_STRING_H
/* begin file include/simdjson/portability.h */
#ifndef SIMDJSON_PORTABILITY_H
#define SIMDJSON_PORTABILITY_H
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#ifdef _MSC_VER
#include <iso646.h>
#endif
#if defined(__x86_64__) || defined(_M_AMD64)
#define IS_X86_64 1
#endif
#if defined(__aarch64__) || defined(_M_ARM64)
#define IS_ARM64 1
#endif
// this is almost standard?
#undef STRINGIFY_IMPLEMENTATION_
#undef STRINGIFY
#define STRINGIFY_IMPLEMENTATION_(a) #a
#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a)
// we are going to use runtime dispatch
#ifdef IS_X86_64
#ifdef __clang__
// clang does not have GCC push pop
// warning: clang attribute push can't be used within a namespace in clang up
// til 8.0 so TARGET_REGION and UNTARGET_REGION must be *outside* of a
// namespace.
#define TARGET_REGION(T) \
_Pragma(STRINGIFY( \
clang attribute push(__attribute__((target(T))), apply_to = function)))
#define UNTARGET_REGION _Pragma("clang attribute pop")
#elif defined(__GNUC__)
// GCC is easier
#define TARGET_REGION(T) \
_Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T)))
#define UNTARGET_REGION _Pragma("GCC pop_options")
#endif // clang then gcc
#endif // x86
// Default target region macros don't do anything.
#ifndef TARGET_REGION
#define TARGET_REGION(T)
#define UNTARGET_REGION
#endif
// under GCC and CLANG, we use these two macros
#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul,lzcnt")
#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul")
#define TARGET_ARM64
// Threading is disabled
#undef SIMDJSON_THREADS_ENABLED
// Is threading enabled?
#if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT)
#define SIMDJSON_THREADS_ENABLED
#endif
#if defined(__clang__)
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
#elif defined(__GNUC__)
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
#else
#define NO_SANITIZE_UNDEFINED
#endif
#ifdef _MSC_VER
#include <intrin.h> // visual studio
#endif
#ifdef _MSC_VER
#define simdjson_strcasecmp _stricmp
#else
#define simdjson_strcasecmp strcasecmp
#endif
namespace simdjson {
// portable version of posix_memalign
static inline void *aligned_malloc(size_t alignment, size_t size) {
void *p;
#ifdef _MSC_VER
p = _aligned_malloc(size, alignment);
#elif defined(__MINGW32__) || defined(__MINGW64__)
p = __mingw_aligned_malloc(size, alignment);
#else
// somehow, if this is used before including "x86intrin.h", it creates an
// implicit defined warning.
if (posix_memalign(&p, alignment, size) != 0) {
return nullptr;
}
#endif
return p;
}
static inline char *aligned_malloc_char(size_t alignment, size_t size) {
return (char *)aligned_malloc(alignment, size);
}
static inline void aligned_free(void *mem_block) {
if (mem_block == nullptr) {
return;
}
#ifdef _MSC_VER
_aligned_free(mem_block);
#elif defined(__MINGW32__) || defined(__MINGW64__)
__mingw_aligned_free(mem_block);
#else
free(mem_block);
#endif
}
static inline void aligned_free_char(char *mem_block) {
aligned_free((void *)mem_block);
}
} // namespace simdjson
#endif // SIMDJSON_PORTABILITY_H
/* end file include/simdjson/portability.h */
/* begin file include/simdjson/common_defs.h */
#ifndef SIMDJSON_COMMON_DEFS_H
#define SIMDJSON_COMMON_DEFS_H
#include <cassert>
// we support documents up to 4GB
#define SIMDJSON_MAXSIZE_BYTES 0xFFFFFFFF
// the input buf should be readable up to buf + SIMDJSON_PADDING
// this is a stopgap; there should be a better description of the
// main loop and its behavior that abstracts over this
// See https://github.com/lemire/simdjson/issues/174
#define SIMDJSON_PADDING 32
#if defined(__GNUC__)
// Marks a block with a name so that MCA analysis can see it.
#define BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name);
#define END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name);
#define DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name);
#else
#define BEGIN_DEBUG_BLOCK(name)
#define END_DEBUG_BLOCK(name)
#define DEBUG_BLOCK(name, block)
#endif
#if !defined(_MSC_VER) && !defined(SIMDJSON_NO_COMPUTED_GOTO)
// Implemented using Labels as Values which works in GCC and CLANG (and maybe
// also in Intel's compiler), but won't work in MSVC.
#define SIMDJSON_USE_COMPUTED_GOTO
#endif
// Align to N-byte boundary
#define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
#define ROUNDDOWN_N(a, n) ((a) & ~((n)-1))
#define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
#ifdef _MSC_VER
#define really_inline __forceinline
#define never_inline __declspec(noinline)
#define UNUSED
#define WARN_UNUSED
#ifndef likely
#define likely(x) x
#endif
#ifndef unlikely
#define unlikely(x) x
#endif
#else
#define really_inline inline __attribute__((always_inline, unused))
#define never_inline inline __attribute__((noinline, unused))
#define UNUSED __attribute__((unused))
#define WARN_UNUSED __attribute__((warn_unused_result))
#ifndef likely
#define likely(x) __builtin_expect(!!(x), 1)
#endif
#ifndef unlikely
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
#endif // MSC_VER
#endif // SIMDJSON_COMMON_DEFS_H
/* end file include/simdjson/common_defs.h */
#include <cstring>
#include <memory>
#include <string>
namespace simdjson {
// low-level function to allocate memory with padding so we can read past the
// "length" bytes safely. if you must provide a pointer to some data, create it
// with this function: length is the max. size in bytes of the string caller is
// responsible to free the memory (free(...))
inline char *allocate_padded_buffer(size_t length) noexcept {
// we could do a simple malloc
// return (char *) malloc(length + SIMDJSON_PADDING);
// However, we might as well align to cache lines...
size_t totalpaddedlength = length + SIMDJSON_PADDING;
char *padded_buffer = aligned_malloc_char(64, totalpaddedlength);
#ifndef NDEBUG
if (padded_buffer == nullptr) {
return nullptr;
}
#endif // NDEBUG
memset(padded_buffer + length, 0, totalpaddedlength - length);
return padded_buffer;
} // allocate_padded_buffer
// Simple string with padded allocation.
// We deliberately forbid copies, users should rely on swap or move
// constructors.
struct padded_string final {
explicit padded_string() noexcept : viable_size(0), data_ptr(nullptr) {}
explicit padded_string(size_t length) noexcept
: viable_size(length), data_ptr(allocate_padded_buffer(length)) {
if (data_ptr != nullptr)
data_ptr[length] = '\0'; // easier when you need a c_str
}
explicit padded_string(const char *data, size_t length) noexcept
: viable_size(length), data_ptr(allocate_padded_buffer(length)) {
if ((data != nullptr) and (data_ptr != nullptr)) {
memcpy(data_ptr, data, length);
data_ptr[length] = '\0'; // easier when you need a c_str
}
}
// note: do not pass std::string arguments by value
padded_string(const std::string & str_ ) noexcept
: viable_size(str_.size()), data_ptr(allocate_padded_buffer(str_.size())) {
if (data_ptr != nullptr) {
memcpy(data_ptr, str_.data(), str_.size());
data_ptr[str_.size()] = '\0'; // easier when you need a c_str
}
}
// note: do pass std::string_view arguments by value
padded_string(std::string_view sv_) noexcept
: viable_size(sv_.size()), data_ptr(allocate_padded_buffer(sv_.size())) {
if (data_ptr != nullptr) {
memcpy(data_ptr, sv_.data(), sv_.size());
data_ptr[sv_.size()] = '\0'; // easier when you need a c_str
}
}
padded_string(padded_string &&o) noexcept
: viable_size(o.viable_size), data_ptr(o.data_ptr) {
o.data_ptr = nullptr; // we take ownership
}
padded_string &operator=(padded_string &&o) {
aligned_free_char(data_ptr);
data_ptr = o.data_ptr;
viable_size = o.viable_size;
o.data_ptr = nullptr; // we take ownership
o.viable_size = 0;
return *this;
}
void swap(padded_string &o) {
size_t tmp_viable_size = viable_size;
char *tmp_data_ptr = data_ptr;
viable_size = o.viable_size;
data_ptr = o.data_ptr;
o.data_ptr = tmp_data_ptr;
o.viable_size = tmp_viable_size;
}
~padded_string() {
aligned_free_char(data_ptr);
}
size_t size() const { return viable_size; }
size_t length() const { return viable_size; }
char *data() const { return data_ptr; }
private:
padded_string &operator=(const padded_string &o) = delete;
padded_string(const padded_string &o) = delete;
size_t viable_size;
char *data_ptr{nullptr};
}; // padded_string
} // namespace simdjson
#endif
/* end file include/simdjson/common_defs.h */
/* begin file include/simdjson/implementation.h */
#ifndef SIMDJSON_IMPLEMENTATION_H
#define SIMDJSON_IMPLEMENTATION_H
#include <optional>
#include <string>
#include <atomic>
#include <vector>
/* begin file include/simdjson/document.h */
#ifndef SIMDJSON_DOCUMENT_H
#define SIMDJSON_DOCUMENT_H
#include <cstring>
#include <memory>
#include <string>
#include <limits>
/* begin file include/simdjson/simdjson.h */
/**
* @file
* @deprecated We'll be removing this file so it isn't confused with the top level simdjson.h
*/
#ifndef SIMDJSON_SIMDJSON_H
#define SIMDJSON_SIMDJSON_H
#endif // SIMDJSON_H
/* end file include/simdjson/simdjson.h */
#define JSON_VALUE_MASK 0x00FFFFFFFFFFFFFF
#define DEFAULT_MAX_DEPTH 1024 // a JSON document with a depth exceeding 1024 is probably de facto invalid
namespace simdjson {
template<size_t max_depth> class document_iterator;
/**
* A parsed JSON document.
*
* This class cannot be copied, only moved, to avoid unintended allocations.
*/
class document {
public:
/**
* Create a document container with zero capacity.
*
* The parser will allocate capacity as needed.
*/
document() noexcept=default;
~document() noexcept=default;
/**
* Take another document's buffers.
*
* @param other The document to take. Its capacity is zeroed and it is invalidated.
*/
document(document &&other) noexcept = default;
document(const document &) = delete; // Disallow copying
/**
* Take another document's buffers.
*
* @param other The document to take. Its capacity is zeroed.
*/
document &operator=(document &&other) noexcept = default;
document &operator=(const document &) = delete; // Disallow copying
// Nested classes
class element;
class array;
class object;
class key_value_pair;
class parser;
template<typename T=element>
class element_result;
class doc_result;
class doc_ref_result;
// Nested classes. See definitions later in file.
using iterator = document_iterator<DEFAULT_MAX_DEPTH>;
/**
* Get the root element of this document as a JSON array.
*/
element root() const noexcept;
/**
* Get the root element of this document as a JSON array.
*/
element_result<array> as_array() const noexcept;
/**
* Get the root element of this document as a JSON object.
*/
element_result<object> as_object() const noexcept;
/**
* Get the root element of this document.
*/
operator element() const noexcept;
/**
* Read the root element of this document as a JSON array.
*
* @return The JSON array.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array
*/
operator array() const noexcept(false);
/**
* Read this element as a JSON object (key/value pairs).
*
* @return The JSON object.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object
*/
operator object() const noexcept(false);
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with the given key, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
element_result<element> operator[](const std::string_view &s) const noexcept;
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
element_result<element> operator[](const char *s) const noexcept;
/**
* Print this JSON to a std::ostream.
*
* @param os the stream to output to.
* @param max_depth the maximum JSON depth to output.
* @return false if the tape is likely wrong (e.g., you did not parse a valid JSON).
*/
bool print_json(std::ostream &os, size_t max_depth=DEFAULT_MAX_DEPTH) const noexcept;
/**
* Dump the raw tape for debugging.
*
* @param os the stream to output to.
* @return false if the tape is likely wrong (e.g., you did not parse a valid JSON).
*/
bool dump_raw_tape(std::ostream &os) const noexcept;
/**
* Parse a JSON document and return a reference to it.
*
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
* those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true,
* it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged
* and copied before parsing.
*
* @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
* realloc_if_needed is true.
* @param len The length of the JSON.
* @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
* @return the document, or an error if the JSON is invalid.
*/
inline static doc_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept;
/**
* Parse a JSON document.
*
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
* those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true,
* it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged
* and copied before parsing.
*
* @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
* realloc_if_needed is true.
* @param len The length of the JSON.
* @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
* @return the document, or an error if the JSON is invalid.
*/
really_inline static doc_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept;
/**
* Parse a JSON document.
*
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
* those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size()
* < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing.
*
* @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or
* a new string will be created with the extra padding.
* @return the document, or an error if the JSON is invalid.
*/
really_inline static doc_result parse(const std::string &s) noexcept;
/**
* Parse a JSON document.
*
* @param s The JSON to parse.
* @return the document, or an error if the JSON is invalid.
*/
really_inline static doc_result parse(const padded_string &s) noexcept;
// We do not want to allow implicit conversion from C string to std::string.
doc_ref_result parse(const char *buf, bool realloc_if_needed = true) noexcept = delete;
std::unique_ptr<uint64_t[]> tape;
std::unique_ptr<uint8_t[]> string_buf;// should be at least byte_capacity
private:
class tape_ref;
enum class tape_type;
bool set_capacity(size_t len);
}; // class document
/**
* A parsed, *owned* document, or an error if the parse failed.
*
* document &doc = document::parse(json);
*
* Returns an owned `document`. When the doc_result (or the document retrieved from it) goes out of
* scope, the document's memory is deallocated.
*
* ## Error Codes vs. Exceptions
*
* This result type allows the user to pick whether to use exceptions or not.
*
* Use like this to avoid exceptions:
*
* auto [doc, error] = document::parse(json);
* if (error) { exit(1); }
*
* Use like this if you'd prefer to use exceptions:
*
* document doc = document::parse(json);
*
*/
class document::doc_result {
public:
/**
* The parsed document. This is *invalid* if there is an error.
*/
document doc;
/**
* The error code, or SUCCESS (0) if there is no error.
*/
error_code error;
/**
* Return the document, or throw an exception if it is invalid.
*
* @return the document.
* @exception invalid_json if the document is invalid or there was an error parsing it.
*/
operator document() noexcept(false);
/**
* Get the error message for the error.
*/
const std::string &get_error_message() const noexcept;
~doc_result() noexcept=default;
private:
doc_result(document &&_doc, error_code _error) noexcept;
doc_result(document &&_doc) noexcept;
doc_result(error_code _error) noexcept;
friend class document;
}; // class document::doc_result
/**
* A parsed document reference, or an error if the parse failed.
*
* document &doc = document::parse(json);
*
* ## Document Ownership
*
* The `document &` refers to an internal document the parser reuses on each `parse()` call. It will
* become invalidated on the next `parse()`.
*
* This is more efficient for common cases where documents are parsed and used one at a time. If you
* need to keep the document around longer, you may *take* it from the parser by casting it:
*
* document doc = parser.parse(); // take ownership
*
* If you do this, the parser will automatically allocate a new document on the next `parse()` call.
*
* ## Error Codes vs. Exceptions
*
* This result type allows the user to pick whether to use exceptions or not.
*
* Use like this to avoid exceptions:
*
* auto [doc, error] = parser.parse(json);
* if (error) { exit(1); }
*
* Use like this if you'd prefer to use exceptions:
*
* document &doc = document::parse(json);
*
*/
class document::doc_ref_result {
public:
/**
* The parsed document. This is *invalid* if there is an error.
*/
document &doc;
/**
* The error code, or SUCCESS (0) if there is no error.
*/
error_code error;
/**
* A reference to the document, or throw an exception if it is invalid.
*
* @return the document.
* @exception invalid_json if the document is invalid or there was an error parsing it.
*/
operator document&() noexcept(false);
/**
* Get the error message for the error.
*/
const std::string &get_error_message() const noexcept;
~doc_ref_result()=default;
private:
doc_ref_result(document &_doc, error_code _error) noexcept;
friend class document::parser;
}; // class document::doc_ref_result
/**
* The possible types in the tape. Internal only.
*/
enum class document::tape_type {
ROOT = 'r',
START_ARRAY = '[',
START_OBJECT = '{',
END_ARRAY = ']',
END_OBJECT = '}',
STRING = '"',
INT64 = 'l',
UINT64 = 'u',
DOUBLE = 'd',
TRUE_VALUE = 't',
FALSE_VALUE = 'f',
NULL_VALUE = 'n'
};
/**
* A reference to an element on the tape. Internal only.
*/
class document::tape_ref {
protected:
really_inline tape_ref() noexcept;
really_inline tape_ref(const document *_doc, size_t _json_index) noexcept;
inline size_t after_element() const noexcept;
really_inline tape_type type() const noexcept;
really_inline uint64_t tape_value() const noexcept;
template<typename T>
really_inline T next_tape_value() const noexcept;
/** The document this element references. */
const document *doc;
/** The index of this element on `doc.tape[]` */
size_t json_index;
friend class document::key_value_pair;
};
/**
* A JSON element.
*
* References an element in a JSON document, representing a JSON null, boolean, string, number,
* array or object.
*/
class document::element : protected document::tape_ref {
public:
/** Whether this element is a json `null`. */
really_inline bool is_null() const noexcept;
/** Whether this is a JSON `true` or `false` */
really_inline bool is_bool() const noexcept;
/** Whether this is a JSON number (e.g. 1, 1.0 or 1e2) */
really_inline bool is_number() const noexcept;
/** Whether this is a JSON integer (e.g. 1 or -1, but *not* 1.0 or 1e2) */
really_inline bool is_integer() const noexcept;
/** Whether this is a JSON string (e.g. "abc") */
really_inline bool is_string() const noexcept;
/** Whether this is a JSON array (e.g. []) */
really_inline bool is_array() const noexcept;
/** Whether this is a JSON array (e.g. []) */
really_inline bool is_object() const noexcept;
/**
* Read this element as a boolean (json `true` or `false`).
*
* @return The boolean value, or:
* - UNEXPECTED_TYPE error if the JSON element is not a boolean
*/
inline element_result<bool> as_bool() const noexcept;
/**
* Read this element as a null-terminated string.
*
* Does *not* convert other types to a string; requires that the JSON type of the element was
* an actual string.
*
* @return A `string_view` into the string, or:
* - UNEXPECTED_TYPE error if the JSON element is not a string
*/
inline element_result<const char *> as_c_str() const noexcept;
/**
* Read this element as a C++ string_view (string with length).
*
* Does *not* convert other types to a string; requires that the JSON type of the element was
* an actual string.
*
* @return A `string_view` into the string, or:
* - UNEXPECTED_TYPE error if the JSON element is not a string
*/
inline element_result<std::string_view> as_string() const noexcept;
/**
* Read this element as an unsigned integer.
*
* @return The uninteger value, or:
* - UNEXPECTED_TYPE if the JSON element is not an integer
* - NUMBER_OUT_OF_RANGE if the integer doesn't fit in 64 bits or is negative
*/
inline element_result<uint64_t> as_uint64_t() const noexcept;
/**
* Read this element as a signed integer.
*
* @return The integer value, or:
* - UNEXPECTED_TYPE if the JSON element is not an integer
* - NUMBER_OUT_OF_RANGE if the integer doesn't fit in 64 bits
*/
inline element_result<int64_t> as_int64_t() const noexcept;
/**
* Read this element as a floating point value.
*
* @return The double value, or:
* - UNEXPECTED_TYPE if the JSON element is not a number
*/
inline element_result<double> as_double() const noexcept;
/**
* Read this element as a JSON array.
*
* @return The array value, or:
* - UNEXPECTED_TYPE if the JSON element is not an array
*/
inline element_result<document::array> as_array() const noexcept;
/**
* Read this element as a JSON object (key/value pairs).
*
* @return The object value, or:
* - UNEXPECTED_TYPE if the JSON element is not an object
*/
inline element_result<document::object> as_object() const noexcept;
/**
* Read this element as a boolean.
*
* @return The boolean value
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a boolean.
*/
inline operator bool() const noexcept(false);
/**
* Read this element as a null-terminated string.
*
* Does *not* convert other types to a string; requires that the JSON type of the element was
* an actual string.
*
* @return The string value.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string.
*/
inline explicit operator const char*() const noexcept(false);
/**
* Read this element as a null-terminated string.
*
* Does *not* convert other types to a string; requires that the JSON type of the element was
* an actual string.
*
* @return The string value.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string.
*/
inline operator std::string_view() const noexcept(false);
/**
* Read this element as an unsigned integer.
*
* @return The integer value.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer
* @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative
*/
inline operator uint64_t() const noexcept(false);
/**
* Read this element as an signed integer.
*
* @return The integer value.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer
* @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits
*/
inline operator int64_t() const noexcept(false);
/**
* Read this element as an double.
*
* @return The double value.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a number
* @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative
*/
inline operator double() const noexcept(false);
/**
* Read this element as a JSON array.
*
* @return The JSON array.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array
*/
inline operator document::array() const noexcept(false);
/**
* Read this element as a JSON object (key/value pairs).
*
* @return The JSON object.
* @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object
*/
inline operator document::object() const noexcept(false);
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
inline element_result<element> operator[](const std::string_view &s) const noexcept;
/**
* Get the value associated with the given key.
*
* Note: The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
inline element_result<element> operator[](const char *s) const noexcept;
private:
really_inline element() noexcept;
really_inline element(const document *_doc, size_t _json_index) noexcept;
friend class document;
template<typename T>
friend class document::element_result;
};
/**
* Represents a JSON array.
*/
class document::array : protected document::tape_ref {
public:
class iterator : tape_ref {
public:
/**
* Get the actual value
*/
inline element operator*() const noexcept;
/**
* Get the next value.
*
* Part of the std::iterator interface.
*/
inline void operator++() noexcept;
/**
* Check if these values come from the same place in the JSON.
*
* Part of the std::iterator interface.
*/
inline bool operator!=(const iterator& other) const noexcept;
private:
really_inline iterator(const document *_doc, size_t _json_index) noexcept;
friend class array;
};
/**
* Return the first array element.
*
* Part of the std::iterable interface.
*/
inline iterator begin() const noexcept;
/**
* One past the last array element.
*
* Part of the std::iterable interface.
*/
inline iterator end() const noexcept;
private:
really_inline array() noexcept;
really_inline array(const document *_doc, size_t _json_index) noexcept;
friend class document::element;
template<typename T>
friend class document::element_result;
};
/**
* Represents a JSON object.
*/
class document::object : protected document::tape_ref {
public:
class iterator : protected document::tape_ref {
public:
/**
* Get the actual key/value pair
*/
inline const document::key_value_pair operator*() const noexcept;
/**
* Get the next key/value pair.
*
* Part of the std::iterator interface.
*/
inline void operator++() noexcept;
/**
* Check if these key value pairs come from the same place in the JSON.
*
* Part of the std::iterator interface.
*/
inline bool operator!=(const iterator& other) const noexcept;
/**
* Get the key of this key/value pair.
*/
inline std::string_view key() const noexcept;
/**
* Get the key of this key/value pair.
*/
inline const char *key_c_str() const noexcept;
/**
* Get the value of this key/value pair.
*/
inline element value() const noexcept;
private:
really_inline iterator(const document *_doc, size_t _json_index) noexcept;
friend class document::object;
};
/**
* Return the first key/value pair.
*
* Part of the std::iterable interface.
*/
inline iterator begin() const noexcept;
/**
* One past the last key/value pair.
*
* Part of the std::iterable interface.
*/
inline iterator end() const noexcept;
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
*/
inline element_result<element> operator[](const std::string_view &s) const noexcept;
/**
* Get the value associated with the given key.
*
* Note: The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
*/
inline element_result<element> operator[](const char *s) const noexcept;
private:
really_inline object() noexcept;
really_inline object(const document *_doc, size_t _json_index) noexcept;
friend class document::element;
template<typename T>
friend class document::element_result;
};
/**
* Key/value pair in an object.
*/
class document::key_value_pair {
public:
std::string_view key;
document::element value;
private:
really_inline key_value_pair(std::string_view _key, document::element _value) noexcept;
friend class document::object;
};
/**
* The result of a JSON navigation or conversion, or an error (if the navigation or conversion
* failed). Allows the user to pick whether to use exceptions or not.
*
* Use like this to avoid exceptions:
*
* auto [str, error] = document::parse(json).root().as_string();
* if (error) { exit(1); }
* cout << str;
*
* Use like this if you'd prefer to use exceptions:
*
* string str = document::parse(json).root();
* cout << str;
*
*/
template<typename T>
class document::element_result {
public:
/** The value */
T value;
/** The error code (or 0 if there is no error) */
error_code error;
inline operator T() const noexcept(false);
private:
really_inline element_result(T value) noexcept;
really_inline element_result(error_code _error) noexcept;
friend class document;
friend class element;
};
// Add exception-throwing navigation / conversion methods to element_result<element>
template<>
class document::element_result<document::element> {
public:
/** The value */
element value;
/** The error code (or 0 if there is no error) */
error_code error;
/** Whether this is a JSON `null` */
inline element_result<bool> is_null() const noexcept;
inline element_result<bool> as_bool() const noexcept;
inline element_result<std::string_view> as_string() const noexcept;
inline element_result<const char *> as_c_str() const noexcept;
inline element_result<uint64_t> as_uint64_t() const noexcept;
inline element_result<int64_t> as_int64_t() const noexcept;
inline element_result<double> as_double() const noexcept;
inline element_result<array> as_array() const noexcept;
inline element_result<object> as_object() const noexcept;
inline operator bool() const noexcept(false);
inline explicit operator const char*() const noexcept(false);
inline operator std::string_view() const noexcept(false);
inline operator uint64_t() const noexcept(false);
inline operator int64_t() const noexcept(false);
inline operator double() const noexcept(false);
inline operator array() const noexcept(false);
inline operator object() const noexcept(false);
inline element_result<element> operator[](const std::string_view &s) const noexcept;
inline element_result<element> operator[](const char *s) const noexcept;
private:
really_inline element_result(element value) noexcept;
really_inline element_result(error_code _error) noexcept;
friend class document;
friend class element;
};
// Add exception-throwing navigation methods to element_result<array>
template<>
class document::element_result<document::array> {
public:
/** The value */
array value;
/** The error code (or 0 if there is no error) */
error_code error;
inline operator array() const noexcept(false);
inline array::iterator begin() const noexcept(false);
inline array::iterator end() const noexcept(false);
private:
really_inline element_result(array value) noexcept;
really_inline element_result(error_code _error) noexcept;
friend class document;
friend class element;
};
// Add exception-throwing navigation methods to element_result<object>
template<>
class document::element_result<document::object> {
public:
/** The value */
object value;
/** The error code (or 0 if there is no error) */
error_code error;
inline operator object() const noexcept(false);
inline object::iterator begin() const noexcept(false);
inline object::iterator end() const noexcept(false);
inline element_result<element> operator[](const std::string_view &s) const noexcept;
inline element_result<element> operator[](const char *s) const noexcept;
private:
really_inline element_result(object value) noexcept;
really_inline element_result(error_code _error) noexcept;
friend class document;
friend class element;
};
/**
* A persistent document parser.
*
* Use this if you intend to parse more than one document. It holds the internal memory necessary
* to do parsing, as well as memory for a single document that is overwritten on each parse.
*
* This class cannot be copied, only moved, to avoid unintended allocations.
*
* @note This is not thread safe: one parser cannot produce two documents at the same time!
*/
class document::parser {
public:
/**
* Create a JSON parser with zero capacity. Call allocate_capacity() to initialize it.
*/
parser()=default;
~parser()=default;
/**
* Take another parser's buffers and state.
*
* @param other The parser to take. Its capacity is zeroed.
*/
parser(document::parser &&other) = default;
parser(const document::parser &) = delete; // Disallow copying
/**
* Take another parser's buffers and state.
*
* @param other The parser to take. Its capacity is zeroed.
*/
parser &operator=(document::parser &&other) = default;
parser &operator=(const document::parser &) = delete; // Disallow copying
/**
* Parse a JSON document and return a reference to it.
*
* The JSON document still lives in the parser: this is the most efficient way to parse JSON
* documents because it reuses the same buffers, but you *must* use the document before you
* destroy the parser or call parse() again.
*
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
* those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true,
* it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged
* and copied before parsing.
*
* @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
* realloc_if_needed is true.
* @param len The length of the JSON.
* @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
* @return the document, or an error if the JSON is invalid.
*/
inline doc_ref_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept;
/**
* Parse a JSON document and return a reference to it.
*
* The JSON document still lives in the parser: this is the most efficient way to parse JSON
* documents because it reuses the same buffers, but you *must* use the document before you
* destroy the parser or call parse() again.
*
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
* those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true,
* it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged
* and copied before parsing.
*
* @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
* realloc_if_needed is true.
* @param len The length of the JSON.
* @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
* @return the document, or an error if the JSON is invalid.
*/
really_inline doc_ref_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept;
/**
* Parse a JSON document and return a reference to it.
*
* The JSON document still lives in the parser: this is the most efficient way to parse JSON
* documents because it reuses the same buffers, but you *must* use the document before you
* destroy the parser or call parse() again.
*
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
* those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size()
* < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing.
*
* @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or
* a new string will be created with the extra padding.
* @return the document, or an error if the JSON is invalid.
*/
really_inline doc_ref_result parse(const std::string &s) noexcept;
/**
* Parse a JSON document and return a reference to it.
*
* The JSON document still lives in the parser: this is the most efficient way to parse JSON
* documents because it reuses the same buffers, but you *must* use the document before you
* destroy the parser or call parse() again.
*
* @param s The JSON to parse.
* @return the document, or an error if the JSON is invalid.
*/
really_inline doc_ref_result parse(const padded_string &s) noexcept;
// We do not want to allow implicit conversion from C string to std::string.
really_inline doc_ref_result parse(const char *buf) noexcept = delete;
/**
* Current capacity: the largest document this parser can support without reallocating.
*/
really_inline size_t capacity() const noexcept;
/**
* The maximum level of nested object and arrays supported by this parser.
*/
really_inline size_t max_depth() const noexcept;
/**
* Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length
* and `max_depth` depth.
*/
WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH);
// type aliases for backcompat
using Iterator = document::iterator;
using InvalidJSON = invalid_json;
// Next location to write to in the tape
uint32_t current_loc{0};
// structural indices passed from stage 1 to stage 2
uint32_t n_structural_indexes{0};
std::unique_ptr<uint32_t[]> structural_indexes;
// location and return address of each open { or [
std::unique_ptr<uint32_t[]> containing_scope_offset;
#ifdef SIMDJSON_USE_COMPUTED_GOTO
std::unique_ptr<void*[]> ret_address;
#else
std::unique_ptr<char[]> ret_address;
#endif
// Next place to write a string
uint8_t *current_string_buf_loc;
bool valid{false};
error_code error{UNINITIALIZED};
// Document we're writing to
document doc;
//
// TODO these are deprecated; use the results of parse instead.
//
// returns true if the document parsed was valid
inline bool is_valid() const noexcept;
// return an error code corresponding to the last parsing attempt, see
// simdjson.h will return UNITIALIZED if no parsing was attempted
inline int get_error_code() const noexcept;
// return the string equivalent of "get_error_code"
inline std::string get_error_message() const noexcept;
// print the json to std::ostream (should be valid)
// return false if the tape is likely wrong (e.g., you did not parse a valid
// JSON).
inline bool print_json(std::ostream &os) const noexcept;
inline bool dump_raw_tape(std::ostream &os) const noexcept;
//
// Parser callbacks: these are internal!
//
// TODO find a way to do this without exposing the interface or crippling performance
//
// this should be called when parsing (right before writing the tapes)
inline void init_stage2() noexcept;
really_inline error_code on_error(error_code new_error_code) noexcept;
really_inline error_code on_success(error_code success_code) noexcept;
really_inline bool on_start_document(uint32_t depth) noexcept;
really_inline bool on_start_object(uint32_t depth) noexcept;
really_inline bool on_start_array(uint32_t depth) noexcept;
// TODO we're not checking this bool
really_inline bool on_end_document(uint32_t depth) noexcept;
really_inline bool on_end_object(uint32_t depth) noexcept;
really_inline bool on_end_array(uint32_t depth) noexcept;
really_inline bool on_true_atom() noexcept;
really_inline bool on_false_atom() noexcept;
really_inline bool on_null_atom() noexcept;
really_inline uint8_t *on_start_string() noexcept;
really_inline bool on_end_string(uint8_t *dst) noexcept;
really_inline bool on_number_s64(int64_t value) noexcept;
really_inline bool on_number_u64(uint64_t value) noexcept;
really_inline bool on_number_double(double value) noexcept;
//
// Called before a parse is initiated.
//
// - Returns CAPACITY if the document is too large
// - Returns MEMALLOC if we needed to allocate memory and could not
//
WARN_UNUSED inline error_code init_parse(size_t len) noexcept;
private:
//
// The maximum document length this parser supports.
//
// Buffers are large enough to handle any document up to this length.
//
size_t _capacity{0};
//
// The maximum depth (number of nested objects and arrays) supported by this parser.
//
// Defaults to DEFAULT_MAX_DEPTH.
//
size_t _max_depth{0};
// all nodes are stored on the doc.tape using a 64-bit word.
//
// strings, double and ints are stored as
// a 64-bit word with a pointer to the actual value
//
//
//
// for objects or arrays, store [ or { at the beginning and } and ] at the
// end. For the openings ([ or {), we annotate them with a reference to the
// location on the doc.tape of the end, and for then closings (} and ]), we
// annotate them with a reference to the location of the opening
//
//
inline void write_tape(uint64_t val, tape_type t) noexcept;
inline void annotate_previous_loc(uint32_t saved_loc, uint64_t val) noexcept;
//
// Set the current capacity: the largest document this parser can support without reallocating.
//
// This will allocate *or deallocate* as necessary.
//
// Returns false if allocation fails.
//
inline WARN_UNUSED bool set_capacity(size_t capacity);
//
// Set the maximum level of nested object and arrays supported by this parser.
//
// This will allocate *or deallocate* as necessary.
//
// Returns false if allocation fails.
//
inline WARN_UNUSED bool set_max_depth(size_t max_depth);
// Used internally to get the document
inline const document &get_document() const noexcept(false);
template<size_t max_depth> friend class document_iterator;
}; // class parser
} // namespace simdjson
#endif // SIMDJSON_DOCUMENT_H
/* end file include/simdjson/simdjson.h */
namespace simdjson {
/**
* An implementation of simdjson for a particular CPU architecture.
*
* Also used to maintain the currently active implementation. The active implementation is
* automatically initialized on first use to the most advanced implementation supported by the host.
*/
class implementation {
public:
/**
* The name of this implementation.
*
* const implementation *impl = simdjson::active_implementation;
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
*
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
*/
virtual const std::string &name() const { return _name; }
/**
* The description of this implementation.
*
* const implementation *impl = simdjson::active_implementation;
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
*
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
*/
virtual const std::string &description() const { return _description; }
/**
* The instruction sets this implementation is compiled against.
*
* @return a mask of all required `instruction_set` values
*/
virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; };
/**
* Run a full document parse (init_parse, stage1 and stage2).
*
* Overridden by each implementation.
*
* @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes.
* @param len the length of the json document.
* @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity.
* @return the error code, or SUCCESS if there was no error.
*/
WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, document::parser &parser) const noexcept = 0;
/**
* Stage 1 of the document parser.
*
* Overridden by each implementation.
*
* @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes.
* @param len the length of the json document.
* @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity.
* @param streaming whether this is being called by a JsonStream parser.
* @return the error code, or SUCCESS if there was no error.
*/
WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept = 0;
/**
* Stage 2 of the document parser.
*
* Overridden by each implementation.
*
* @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes.
* @param len the length of the json document.
* @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity.
* @return the error code, or SUCCESS if there was no error.
*/
WARN_UNUSED virtual error_code stage2(const uint8_t *buf, size_t len, document::parser &parser) const noexcept = 0;
/**
* Stage 2 of the document parser for JsonStream.
*
* Overridden by each implementation.
*
* @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes.
* @param len the length of the json document.
* @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity.
* @param next_json the next structural index. Start this at 0 the first time, and it will be updated to the next value to pass each time.
* @return the error code, SUCCESS if there was no error, or SUCCESS_AND_HAS_MORE if there was no error and stage2 can be called again.
*/
WARN_UNUSED virtual error_code stage2(const uint8_t *buf, size_t len, document::parser &parser, size_t &next_json) const noexcept = 0;
protected:
really_inline implementation(
const std::string &name,
const std::string &description,
uint32_t required_instruction_sets
) :
_name(name),
_description(description),
_required_instruction_sets(required_instruction_sets)
{
}
private:
/**
* The name of this implementation.
*/
const std::string _name;
/**
* The description of this implementation.
*/
const std::string _description;
/**
* Instruction sets required for this implementation.
*/
const uint32_t _required_instruction_sets;
};
namespace internal {
/**
* The list of available implementations compiled into simdjson.
*/
class available_implementation_list {
public:
/** Get the list of available implementations compiled into simdjson */
really_inline available_implementation_list() {}
/** Number of implementations */
size_t size() const noexcept;
/** STL const begin() iterator */
const implementation * const *begin() const noexcept;
/** STL const end() iterator */
const implementation * const *end() const noexcept;
/**
* Get the implementation with the given name.
*
* Case sensitive.
*
* const implementation *impl = simdjson::available_implementations["westmere"];
* if (!impl) { exit(1); }
* simdjson::active_implementation = impl;
*
* @param name the implementation to find, e.g. "westmere", "haswell", "arm64"
* @return the implementation, or nullptr if the parse failed.
*/
const implementation * operator[](const std::string& name) const noexcept {
for (const implementation * impl : *this) {
if (impl->name() == name) { return impl; }
}
return nullptr;
}
/**
* Detect the most advanced implementation supported by the current host.
*
* This is used to initialize the implementation on startup.
*
* const implementation *impl = simdjson::available_implementation::detect_best_supported();
* simdjson::active_implementation = impl;
*
* @return the most advanced supported implementation for the current host, or an
* implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported
* implementation. Will never return nullptr.
*/
const implementation *detect_best_supported() const noexcept;
};
// Detects best supported implementation on first use, and sets it
class detect_best_supported_implementation_on_first_use final : public implementation {
public:
const std::string& name() const noexcept final { return set_best()->name(); }
const std::string& description() const noexcept final { return set_best()->description(); }
uint32_t required_instruction_sets() const noexcept final { return set_best()->required_instruction_sets(); }
WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, document::parser &parser) const noexcept final {
return set_best()->parse(buf, len, parser);
}
WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept final {
return set_best()->stage1(buf, len, parser, streaming);
}
WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, document::parser &parser) const noexcept final {
return set_best()->stage2(buf, len, parser);
}
WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, document::parser &parser, size_t &next_json) const noexcept final {
return set_best()->stage2(buf, len, parser, next_json);
}
really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {}
private:
const implementation *set_best() const noexcept;
};
inline const detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton;
template<typename T>
class atomic_ptr {
public:
atomic_ptr(T *_ptr) : ptr{_ptr} {}
operator const T*() const { return ptr.load(); }
const T& operator*() const { return *ptr; }
const T* operator->() const { return ptr.load(); }
operator T*() { return ptr.load(); }
T& operator*() { return *ptr; }
T* operator->() { return ptr.load(); }
T* operator=(T *_ptr) { return ptr = _ptr; }
private:
std::atomic<T*> ptr;
};
} // namespace [simdjson::]internal
/**
* The list of available implementations compiled into simdjson.
*/
inline const internal::available_implementation_list available_implementations;
/**
* The active implementation.
*
* Automatically initialized on first use to the most advanced implementation supported by this hardware.
*/
inline internal::atomic_ptr<const implementation> active_implementation = &internal::detect_best_supported_implementation_on_first_use_singleton;
} // namespace simdjson
#endif // SIMDJSON_IMPLEMENTATION_H
/* end file include/simdjson/simdjson.h */
/* begin file include/simdjson/jsonstream.h */
#ifndef SIMDJSON_JSONSTREAM_H
#define SIMDJSON_JSONSTREAM_H
#include <thread>
namespace simdjson {
/*************************************************************************************
* The main motivation for this piece of software is to achieve maximum speed
*and offer
* good quality of life while parsing files containing multiple JSON documents.
*
* Since we want to offer flexibility and not restrict ourselves to a specific
*file
* format, we support any file that contains any valid JSON documents separated
*by one
* or more character that is considered a whitespace by the JSON spec.
* Namely: space, nothing, linefeed, carriage return, horizontal tab.
* Anything that is not whitespace will be parsed as a JSON document and could
*lead
* to failure.
*
* To offer maximum parsing speed, our implementation processes the data inside
*the
* buffer by batches and their size is defined by the parameter "batch_size".
* By loading data in batches, we can optimize the time spent allocating data in
*the
* parser and can also open the possibility of multi-threading.
* The batch_size must be at least as large as the biggest document in the file,
*but
* not too large in order to submerge the chached memory. We found that 1MB is
* somewhat a sweet spot for now. Eventually, this batch_size could be fully
* automated and be optimal at all times.
************************************************************************************/
/**
* The template parameter (string_container) must
* support the data() and size() methods, returning a pointer
* to a char* and to the number of bytes respectively.
* The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end
* of the string, so if you do not use a padded_string container,
* you have the responsability to overallocated. If you fail to
* do so, your software may crash if you cross a page boundary,
* and you should expect memory checkers to object.
* Most users should use a simdjson::padded_string.
*/
template <class string_container = padded_string> class JsonStream {
public:
/* Create a JsonStream object that can be used to parse sequentially the valid
* JSON documents found in the buffer "buf".
*
* The batch_size must be at least as large as the biggest document in the
* file, but
* not too large to submerge the cached memory. We found that 1MB is
* somewhat a sweet spot for now.
*
* The user is expected to call the following json_parse method to parse the
* next
* valid JSON document found in the buffer. This method can and is expected
* to be
* called in a loop.
*
* Various methods are offered to keep track of the status, like
* get_current_buffer_loc,
* get_n_parsed_docs, get_n_bytes_parsed, etc.
*
* */
JsonStream(const string_container &s, size_t batch_size = 1000000);
~JsonStream();
/* Parse the next document found in the buffer previously given to JsonStream.
* The content should be a valid JSON document encoded as UTF-8. If there is a
* UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
* discouraged.
*
* You do NOT need to pre-allocate a parser. This function takes care of
* pre-allocating a capacity defined by the batch_size defined when creating
the
* JsonStream object.
*
* The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in
case
* of success and indicates that the buffer still contains more data to be
parsed,
* meaning this function can be called again to return the next JSON document
* after this one.
*
* The function returns simdjson::SUCCESS (as integer = 0) in case of success
* and indicates that the buffer has successfully been parsed to the end.
* Every document it contained has been parsed without error.
*
* The function returns an error code from simdjson/simdjson.h in case of
failure
* such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and
so forth;
* the simdjson::error_message function converts these error codes into a
* string).
*
* You can also check validity by calling parser.is_valid(). The same parser
can
* and should be reused for the other documents in the buffer. */
int json_parse(document::parser &parser);
/* Returns the location (index) of where the next document should be in the
* buffer.
* Can be used for debugging, it tells the user the position of the end of the
* last
* valid JSON document parsed*/
inline size_t get_current_buffer_loc() const { return current_buffer_loc; }
/* Returns the total amount of complete documents parsed by the JsonStream,
* in the current buffer, at the given time.*/
inline size_t get_n_parsed_docs() const { return n_parsed_docs; }
/* Returns the total amount of data (in bytes) parsed by the JsonStream,
* in the current buffer, at the given time.*/
inline size_t get_n_bytes_parsed() const { return n_bytes_parsed; }
private:
inline const uint8_t *buf() const { return reinterpret_cast<uint8_t*>(str.data()) + str_start; }
inline void advance(size_t offset) { str_start += offset; }
inline size_t remaining() const { return str.size() - str_start; }
const string_container &str;
size_t _batch_size; // this is actually variable!
size_t str_start{0};
size_t next_json{0};
bool load_next_batch{true};
size_t current_buffer_loc{0};
#ifdef SIMDJSON_THREADS_ENABLED
size_t last_json_buffer_loc{0};
#endif
size_t n_parsed_docs{0};
size_t n_bytes_parsed{0};
simdjson::implementation *stage_parser;
#ifdef SIMDJSON_THREADS_ENABLED
error_code stage1_is_ok_thread{SUCCESS};
std::thread stage_1_thread;
document::parser parser_thread;
#endif
}; // end of class JsonStream
} // end of namespace simdjson
#endif // SIMDJSON_JSONSTREAM_H
/* end file include/simdjson/jsonstream.h */
/* begin file include/simdjson/jsonminifier.h */
#ifndef SIMDJSON_JSONMINIFIER_H
#define SIMDJSON_JSONMINIFIER_H
#include <cstddef>
#include <cstdint>
#include <string_view>
namespace simdjson {
// Take input from buf and remove useless whitespace, write it to out; buf and
// out can be the same pointer. Result is null terminated,
// return the string length (minus the null termination).
// The accelerated version of this function only runs on AVX2 hardware.
size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out);
static inline size_t json_minify(const char *buf, size_t len, char *out) {
return json_minify(reinterpret_cast<const uint8_t *>(buf), len,
reinterpret_cast<uint8_t *>(out));
}
static inline size_t json_minify(const std::string_view &p, char *out) {
return json_minify(p.data(), p.size(), out);
}
static inline size_t json_minify(const padded_string &p, char *out) {
return json_minify(p.data(), p.size(), out);
}
} // namespace simdjson
#endif
/* end file include/simdjson/jsonminifier.h */
// Deprecated API
/* begin file include/simdjson/parsedjsoniterator.h */
// TODO Remove this -- deprecated API and files
#ifndef SIMDJSON_PARSEDJSONITERATOR_H
#define SIMDJSON_PARSEDJSONITERATOR_H
/* begin file include/simdjson/document_iterator.h */
#ifndef SIMDJSON_DOCUMENT_ITERATOR_H
#define SIMDJSON_DOCUMENT_ITERATOR_H
#include <cstring>
#include <string>
#include <iostream>
#include <iterator>
#include <limits>
#include <stdexcept>
/* begin file include/simdjson/internal/jsonformatutils.h */
#ifndef SIMDJSON_INTERNAL_JSONFORMATUTILS_H
#define SIMDJSON_INTERNAL_JSONFORMATUTILS_H
#include <iomanip>
#include <iostream>
namespace simdjson::internal {
// ends with zero char
static inline void print_with_escapes(const unsigned char *src, std::ostream &os) {
while (*src) {
switch (*src) {
case '\b':
os << '\\';
os << 'b';
break;
case '\f':
os << '\\';
os << 'f';
break;
case '\n':
os << '\\';
os << 'n';
break;
case '\r':
os << '\\';
os << 'r';
break;
case '\"':
os << '\\';
os << '"';
break;
case '\t':
os << '\\';
os << 't';
break;
case '\\':
os << '\\';
os << '\\';
break;
default:
if (*src <= 0x1F) {
std::ios::fmtflags f(os.flags());
os << std::hex << std::setw(4) << std::setfill('0')
<< static_cast<int>(*src);
os.flags(f);
} else {
os << *src;
}
}
src++;
}
}
// print len chars
static inline void print_with_escapes(const unsigned char *src,
std::ostream &os, size_t len) {
const unsigned char *finalsrc = src + len;
while (src < finalsrc) {
switch (*src) {
case '\b':
os << '\\';
os << 'b';
break;
case '\f':
os << '\\';
os << 'f';
break;
case '\n':
os << '\\';
os << 'n';
break;
case '\r':
os << '\\';
os << 'r';
break;
case '\"':
os << '\\';
os << '"';
break;
case '\t':
os << '\\';
os << 't';
break;
case '\\':
os << '\\';
os << '\\';
break;
default:
if (*src <= 0x1F) {
std::ios::fmtflags f(os.flags());
os << std::hex << std::setw(4) << std::setfill('0')
<< static_cast<int>(*src);
os.flags(f);
} else {
os << *src;
}
}
src++;
}
}
static inline void print_with_escapes(const char *src, std::ostream &os) {
print_with_escapes(reinterpret_cast<const unsigned char *>(src), os);
}
static inline void print_with_escapes(const char *src, std::ostream &os, size_t len) {
print_with_escapes(reinterpret_cast<const unsigned char *>(src), os, len);
}
} // namespace simdjson::internal
#endif // SIMDJSON_INTERNAL_JSONFORMATUTILS_H
/* end file include/simdjson/internal/jsonformatutils.h */
namespace simdjson {
template <size_t max_depth> class document_iterator {
public:
document_iterator(const document::parser &parser);
document_iterator(const document &doc) noexcept;
document_iterator(const document_iterator &o) noexcept;
document_iterator &operator=(const document_iterator &o) noexcept;
inline bool is_ok() const;
// useful for debuging purposes
inline size_t get_tape_location() const;
// useful for debuging purposes
inline size_t get_tape_length() const;
// returns the current depth (start at 1 with 0 reserved for the fictitious
// root node)
inline size_t get_depth() const;
// A scope is a series of nodes at the same depth, typically it is either an
// object ({) or an array ([). The root node has type 'r'.
inline uint8_t get_scope_type() const;
// move forward in document order
inline bool move_forward();
// retrieve the character code of what we're looking at:
// [{"slutfn are the possibilities
inline uint8_t get_type() const {
return current_type; // short functions should be inlined!
}
// get the int64_t value at this node; valid only if get_type is "l"
inline int64_t get_integer() const {
if (location + 1 >= tape_length) {
return 0; // default value in case of error
}
return static_cast<int64_t>(doc.tape[location + 1]);
}
// get the value as uint64; valid only if if get_type is "u"
inline uint64_t get_unsigned_integer() const {
if (location + 1 >= tape_length) {
return 0; // default value in case of error
}
return doc.tape[location + 1];
}
// get the string value at this node (NULL ended); valid only if get_type is "
// note that tabs, and line endings are escaped in the returned value (see
// print_with_escapes) return value is valid UTF-8, it may contain NULL chars
// within the string: get_string_length determines the true string length.
inline const char *get_string() const {
return reinterpret_cast<const char *>(
doc.string_buf.get() + (current_val & JSON_VALUE_MASK) + sizeof(uint32_t));
}
// return the length of the string in bytes
inline uint32_t get_string_length() const {
uint32_t answer;
memcpy(&answer,
reinterpret_cast<const char *>(doc.string_buf.get() +
(current_val & JSON_VALUE_MASK)),
sizeof(uint32_t));
return answer;
}
// get the double value at this node; valid only if
// get_type() is "d"
inline double get_double() const {
if (location + 1 >= tape_length) {
return std::numeric_limits<double>::quiet_NaN(); // default value in
// case of error
}
double answer;
memcpy(&answer, &doc.tape[location + 1], sizeof(answer));
return answer;
}
inline bool is_object_or_array() const { return is_object() || is_array(); }
inline bool is_object() const { return get_type() == '{'; }
inline bool is_array() const { return get_type() == '['; }
inline bool is_string() const { return get_type() == '"'; }
// Returns true if the current type of node is an signed integer.
// You can get its value with `get_integer()`.
inline bool is_integer() const { return get_type() == 'l'; }
// Returns true if the current type of node is an unsigned integer.
// You can get its value with `get_unsigned_integer()`.
//
// NOTE:
// Only a large value, which is out of range of a 64-bit signed integer, is
// represented internally as an unsigned node. On the other hand, a typical
// positive integer, such as 1, 42, or 1000000, is as a signed node.
// Be aware this function returns false for a signed node.
inline bool is_unsigned_integer() const { return get_type() == 'u'; }
inline bool is_double() const { return get_type() == 'd'; }
inline bool is_number() const {
return is_integer() || is_unsigned_integer() || is_double();
}
inline bool is_true() const { return get_type() == 't'; }
inline bool is_false() const { return get_type() == 'f'; }
inline bool is_null() const { return get_type() == 'n'; }
static bool is_object_or_array(uint8_t type) {
return ((type == '[') || (type == '{'));
}
// when at {, go one level deep, looking for a given key
// if successful, we are left pointing at the value,
// if not, we are still pointing at the object ({)
// (in case of repeated keys, this only finds the first one).
// We seek the key using C's strcmp so if your JSON strings contain
// NULL chars, this would trigger a false positive: if you expect that
// to be the case, take extra precautions.
// Furthermore, we do the comparison character-by-character
// without taking into account Unicode equivalence.
inline bool move_to_key(const char *key);
// as above, but case insensitive lookup (strcmpi instead of strcmp)
inline bool move_to_key_insensitive(const char *key);
// when at {, go one level deep, looking for a given key
// if successful, we are left pointing at the value,
// if not, we are still pointing at the object ({)
// (in case of repeated keys, this only finds the first one).
// The string we search for can contain NULL values.
// Furthermore, we do the comparison character-by-character
// without taking into account Unicode equivalence.
inline bool move_to_key(const char *key, uint32_t length);
// when at a key location within an object, this moves to the accompanying
// value (located next to it). This is equivalent but much faster than
// calling "next()".
inline void move_to_value();
// when at [, go one level deep, and advance to the given index.
// if successful, we are left pointing at the value,
// if not, we are still pointing at the array ([)
inline bool move_to_index(uint32_t index);
// Moves the iterator to the value correspoding to the json pointer.
// Always search from the root of the document.
// if successful, we are left pointing at the value,
// if not, we are still pointing the same value we were pointing before the
// call. The json pointer follows the rfc6901 standard's syntax:
// https://tools.ietf.org/html/rfc6901 However, the standard says "If a
// referenced member name is not unique in an object, the member that is
// referenced is undefined, and evaluation fails". Here we just return the
// first corresponding value. The length parameter is the length of the
// jsonpointer string ('pointer').
bool move_to(const char *pointer, uint32_t length);
// Moves the iterator to the value correspoding to the json pointer.
// Always search from the root of the document.
// if successful, we are left pointing at the value,
// if not, we are still pointing the same value we were pointing before the
// call. The json pointer implementation follows the rfc6901 standard's
// syntax: https://tools.ietf.org/html/rfc6901 However, the standard says
// "If a referenced member name is not unique in an object, the member that
// is referenced is undefined, and evaluation fails". Here we just return
// the first corresponding value.
inline bool move_to(const std::string &pointer) {
return move_to(pointer.c_str(), pointer.length());
}
private:
// Almost the same as move_to(), except it searchs from the current
// position. The pointer's syntax is identical, though that case is not
// handled by the rfc6901 standard. The '/' is still required at the
// beginning. However, contrary to move_to(), the URI Fragment Identifier
// Representation is not supported here. Also, in case of failure, we are
// left pointing at the closest value it could reach. For these reasons it
// is private. It exists because it is used by move_to().
bool relative_move_to(const char *pointer, uint32_t length);
public:
// throughout return true if we can do the navigation, false
// otherwise
// Withing a given scope (series of nodes at the same depth within either an
// array or an object), we move forward.
// Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, {
// and [. At the object ({) or at the array ([), you can issue a "down" to
// visit their content. valid if we're not at the end of a scope (returns
// true).
inline bool next();
// Within a given scope (series of nodes at the same depth within either an
// array or an object), we move backward.
// Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true
// when starting at the end of the scope. At the object ({) or at the array
// ([), you can issue a "down" to visit their content.
// Performance warning: This function is implemented by starting again
// from the beginning of the scope and scanning forward. You should expect
// it to be relatively slow.
inline bool prev();
// Moves back to either the containing array or object (type { or [) from
// within a contained scope.
// Valid unless we are at the first level of the document
inline bool up();
// Valid if we're at a [ or { and it starts a non-empty scope; moves us to
// start of that deeper scope if it not empty. Thus, given [true, null,
// {"a":1}, [1,2]], if we are at the { node, we would move to the "a" node.
inline bool down();
// move us to the start of our current scope,
// a scope is a series of nodes at the same level
inline void to_start_scope();
inline void rewind() {
while (up())
;
}
// void to_end_scope(); // move us to
// the start of our current scope; always succeeds
// print the node we are currently pointing at
bool print(std::ostream &os, bool escape_strings = true) const;
typedef struct {
size_t start_of_scope;
uint8_t scope_type;
} scopeindex_t;
private:
const document &doc;
size_t depth;
size_t location; // our current location on a tape
size_t tape_length;
uint8_t current_type;
uint64_t current_val;
scopeindex_t depth_index[max_depth];
};
} // namespace simdjson
#endif // SIMDJSON_DOCUMENT_ITERATOR_H
/* end file include/simdjson/internal/jsonformatutils.h */
#endif
/* end file include/simdjson/internal/jsonformatutils.h */
/* begin file include/simdjson/jsonparser.h */
// TODO Remove this -- deprecated API and files
#ifndef SIMDJSON_JSONPARSER_H
#define SIMDJSON_JSONPARSER_H
/* begin file include/simdjson/parsedjson.h */
// TODO Remove this -- deprecated API and files
#ifndef SIMDJSON_PARSEDJSON_H
#define SIMDJSON_PARSEDJSON_H
namespace simdjson {
using ParsedJson = document::parser;
} // namespace simdjson
#endif
/* end file include/simdjson/parsedjson.h */
/* begin file include/simdjson/jsonioutil.h */
#ifndef SIMDJSON_JSONIOUTIL_H
#define SIMDJSON_JSONIOUTIL_H
#include <exception>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
namespace simdjson {
// load a file in memory...
// get a corpus; pad out to cache line so we can always use SIMD
// throws exceptions in case of failure
// first element of the pair is a string (null terminated)
// whereas the second element is the length.
// caller is responsible to free (aligned_free((void*)result.data())))
//
// throws an exception if the file cannot be opened, use try/catch
// try {
// p = get_corpus(filename);
// } catch (const std::exception& e) {
// aligned_free((void*)p.data());
// std::cout << "Could not load the file " << filename << std::endl;
// }
padded_string get_corpus(const std::string &filename);
} // namespace simdjson
#endif
/* end file include/simdjson/jsonioutil.h */
namespace simdjson {
//
// C API (json_parse and build_parsed_json) declarations
//
inline int json_parse(const uint8_t *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept {
error_code code = parser.parse(buf, len, realloc_if_needed).error;
// The deprecated json_parse API is a signal that the user plans to *use* the error code / valid
// bits in the parser instead of heeding the result code. The normal parser unsets those in
// anticipation of making the error code ephemeral.
// Here we put the code back into the parser, until we've removed this method.
parser.valid = code == SUCCESS;
parser.error = code;
return code;
}
inline int json_parse(const char *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept {
return json_parse(reinterpret_cast<const uint8_t *>(buf), len, parser, realloc_if_needed);
}
inline int json_parse(const std::string &s, document::parser &parser, bool realloc_if_needed = true) noexcept {
return json_parse(s.data(), s.length(), parser, realloc_if_needed);
}
inline int json_parse(const padded_string &s, document::parser &parser) noexcept {
return json_parse(s.data(), s.length(), parser, false);
}
WARN_UNUSED static document::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept {
document::parser parser;
if (!parser.allocate_capacity(len)) {
parser.valid = false;
parser.error = MEMALLOC;
return parser;
}
json_parse(buf, len, parser, realloc_if_needed);
return parser;
}
WARN_UNUSED inline document::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept {
return build_parsed_json(reinterpret_cast<const uint8_t *>(buf), len, realloc_if_needed);
}
WARN_UNUSED inline document::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept {
return build_parsed_json(s.data(), s.length(), realloc_if_needed);
}
WARN_UNUSED inline document::parser build_parsed_json(const padded_string &s) noexcept {
return build_parsed_json(s.data(), s.length(), false);
}
// We do not want to allow implicit conversion from C string to std::string.
int json_parse(const char *buf, document::parser &parser) noexcept = delete;
document::parser build_parsed_json(const char *buf) noexcept = delete;
} // namespace simdjson
#endif
/* end file include/simdjson/jsonioutil.h */
// Inline functions
/* begin file include/simdjson/inline/document.h */
#ifndef SIMDJSON_INLINE_DOCUMENT_H
#define SIMDJSON_INLINE_DOCUMENT_H
// Inline implementations go in here.
#include <iostream>
namespace simdjson {
//
// document::element_result<T> inline implementation
//
template<typename T>
inline document::element_result<T>::element_result(T _value) noexcept : value(_value), error{SUCCESS} {}
template<typename T>
inline document::element_result<T>::element_result(error_code _error) noexcept : value(), error{_error} {}
template<>
inline document::element_result<std::string_view>::operator std::string_view() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
template<>
inline document::element_result<const char *>::operator const char *() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
template<>
inline document::element_result<bool>::operator bool() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
template<>
inline document::element_result<uint64_t>::operator uint64_t() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
template<>
inline document::element_result<int64_t>::operator int64_t() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
template<>
inline document::element_result<double>::operator double() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
//
// document::element_result<document::array> inline implementation
//
inline document::element_result<document::array>::element_result(document::array _value) noexcept : value(_value), error{SUCCESS} {}
inline document::element_result<document::array>::element_result(error_code _error) noexcept : value(), error{_error} {}
inline document::element_result<document::array>::operator document::array() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
inline document::array::iterator document::element_result<document::array>::begin() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value.begin();
}
inline document::array::iterator document::element_result<document::array>::end() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value.end();
}
//
// document::element_result<document::object> inline implementation
//
inline document::element_result<document::object>::element_result(document::object _value) noexcept : value(_value), error{SUCCESS} {}
inline document::element_result<document::object>::element_result(error_code _error) noexcept : value(), error{_error} {}
inline document::element_result<document::object>::operator document::object() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value;
}
inline document::element_result<document::element> document::element_result<document::object>::operator[](const std::string_view &key) const noexcept {
if (error) { return error; }
return value[key];
}
inline document::element_result<document::element> document::element_result<document::object>::operator[](const char *key) const noexcept {
if (error) { return error; }
return value[key];
}
inline document::object::iterator document::element_result<document::object>::begin() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value.begin();
}
inline document::object::iterator document::element_result<document::object>::end() const noexcept(false) {
if (error) { throw invalid_json(error); }
return value.end();
}
//
// document::element_result<document::element> inline implementation
//
inline document::element_result<document::element>::element_result(document::element _value) noexcept : value(_value), error{SUCCESS} {}
inline document::element_result<document::element>::element_result(error_code _error) noexcept : value(), error{_error} {}
inline document::element_result<bool> document::element_result<document::element>::is_null() const noexcept {
if (error) { return error; }
return value.is_null();
}
inline document::element_result<bool> document::element_result<document::element>::as_bool() const noexcept {
if (error) { return error; }
return value.as_bool();
}
inline document::element_result<const char*> document::element_result<document::element>::as_c_str() const noexcept {
if (error) { return error; }
return value.as_c_str();
}
inline document::element_result<std::string_view> document::element_result<document::element>::as_string() const noexcept {
if (error) { return error; }
return value.as_string();
}
inline document::element_result<uint64_t> document::element_result<document::element>::as_uint64_t() const noexcept {
if (error) { return error; }
return value.as_uint64_t();
}
inline document::element_result<int64_t> document::element_result<document::element>::as_int64_t() const noexcept {
if (error) { return error; }
return value.as_int64_t();
}
inline document::element_result<double> document::element_result<document::element>::as_double() const noexcept {
if (error) { return error; }
return value.as_double();
}
inline document::element_result<document::array> document::element_result<document::element>::as_array() const noexcept {
if (error) { return error; }
return value.as_array();
}
inline document::element_result<document::object> document::element_result<document::element>::as_object() const noexcept {
if (error) { return error; }
return value.as_object();
}
inline document::element_result<document::element>::operator bool() const noexcept(false) {
return as_bool();
}
inline document::element_result<document::element>::operator const char *() const noexcept(false) {
return as_c_str();
}
inline document::element_result<document::element>::operator std::string_view() const noexcept(false) {
return as_string();
}
inline document::element_result<document::element>::operator uint64_t() const noexcept(false) {
return as_uint64_t();
}
inline document::element_result<document::element>::operator int64_t() const noexcept(false) {
return as_int64_t();
}
inline document::element_result<document::element>::operator double() const noexcept(false) {
return as_double();
}
inline document::element_result<document::element>::operator document::array() const noexcept(false) {
return as_array();
}
inline document::element_result<document::element>::operator document::object() const noexcept(false) {
return as_object();
}
inline document::element_result<document::element> document::element_result<document::element>::operator[](const std::string_view &key) const noexcept {
if (error) { return *this; }
return value[key];
}
inline document::element_result<document::element> document::element_result<document::element>::operator[](const char *key) const noexcept {
if (error) { return *this; }
return value[key];
}
//
// document inline implementation
//
inline document::element document::root() const noexcept {
return document::element(this, 1);
}
inline document::element_result<document::array> document::as_array() const noexcept {
return root().as_array();
}
inline document::element_result<document::object> document::as_object() const noexcept {
return root().as_object();
}
inline document::operator document::element() const noexcept {
return root();
}
inline document::operator document::array() const noexcept(false) {
return root();
}
inline document::operator document::object() const noexcept(false) {
return root();
}
inline document::element_result<document::element> document::operator[](const std::string_view &key) const noexcept {
return root()[key];
}
inline document::element_result<document::element> document::operator[](const char *key) const noexcept {
return root()[key];
}
inline document::doc_result document::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept {
document::parser parser;
if (!parser.allocate_capacity(len)) {
return MEMALLOC;
}
auto [doc, error] = parser.parse(buf, len, realloc_if_needed);
return document::doc_result((document &&)doc, error);
}
really_inline document::doc_result document::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept {
return parse((const uint8_t *)buf, len, realloc_if_needed);
}
really_inline document::doc_result document::parse(const std::string &s) noexcept {
return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
}
really_inline document::doc_result document::parse(const padded_string &s) noexcept {
return parse(s.data(), s.length(), false);
}
WARN_UNUSED
inline bool document::set_capacity(size_t capacity) {
if (capacity == 0) {
string_buf.reset();
tape.reset();
return true;
}
// a pathological input like "[[[[..." would generate len tape elements, so
// need a capacity of at least len + 1, but it is also possible to do
// worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"
//where len + 1 tape elements are
// generated, see issue https://github.com/lemire/simdjson/issues/345
size_t tape_capacity = ROUNDUP_N(capacity + 2, 64);
// a document with only zero-length strings... could have len/3 string
// and we would need len/3 * 5 bytes on the string buffer
size_t string_capacity = ROUNDUP_N(5 * capacity / 3 + 32, 64);
string_buf.reset( new (std::nothrow) uint8_t[string_capacity]);
tape.reset(new (std::nothrow) uint64_t[tape_capacity]);
return string_buf && tape;
}
inline bool document::print_json(std::ostream &os, size_t max_depth) const noexcept {
uint32_t string_length;
size_t tape_idx = 0;
uint64_t tape_val = tape[tape_idx];
uint8_t type = (tape_val >> 56);
size_t how_many = 0;
if (type == 'r') {
how_many = tape_val & JSON_VALUE_MASK;
} else {
// Error: no starting root node?
return false;
}
tape_idx++;
std::unique_ptr<bool[]> in_object(new bool[max_depth]);
std::unique_ptr<size_t[]> in_object_idx(new size_t[max_depth]);
int depth = 1; // only root at level 0
in_object_idx[depth] = 0;
in_object[depth] = false;
for (; tape_idx < how_many; tape_idx++) {
tape_val = tape[tape_idx];
uint64_t payload = tape_val & JSON_VALUE_MASK;
type = (tape_val >> 56);
if (!in_object[depth]) {
if ((in_object_idx[depth] > 0) && (type != ']')) {
os << ",";
}
in_object_idx[depth]++;
} else { // if (in_object) {
if ((in_object_idx[depth] > 0) && ((in_object_idx[depth] & 1) == 0) &&
(type != '}')) {
os << ",";
}
if (((in_object_idx[depth] & 1) == 1)) {
os << ":";
}
in_object_idx[depth]++;
}
switch (type) {
case '"': // we have a string
os << '"';
memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t));
internal::print_with_escapes(
(const unsigned char *)(string_buf.get() + payload + sizeof(uint32_t)),
os, string_length);
os << '"';
break;
case 'l': // we have a long int
if (tape_idx + 1 >= how_many) {
return false;
}
os << static_cast<int64_t>(tape[++tape_idx]);
break;
case 'u':
if (tape_idx + 1 >= how_many) {
return false;
}
os << tape[++tape_idx];
break;
case 'd': // we have a double
if (tape_idx + 1 >= how_many) {
return false;
}
double answer;
memcpy(&answer, &tape[++tape_idx], sizeof(answer));
os << answer;
break;
case 'n': // we have a null
os << "null";
break;
case 't': // we have a true
os << "true";
break;
case 'f': // we have a false
os << "false";
break;
case '{': // we have an object
os << '{';
depth++;
in_object[depth] = true;
in_object_idx[depth] = 0;
break;
case '}': // we end an object
depth--;
os << '}';
break;
case '[': // we start an array
os << '[';
depth++;
in_object[depth] = false;
in_object_idx[depth] = 0;
break;
case ']': // we end an array
depth--;
os << ']';
break;
case 'r': // we start and end with the root node
// should we be hitting the root node?
return false;
default:
// bug?
return false;
}
}
return true;
}
inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
uint32_t string_length;
size_t tape_idx = 0;
uint64_t tape_val = tape[tape_idx];
uint8_t type = (tape_val >> 56);
os << tape_idx << " : " << type;
tape_idx++;
size_t how_many = 0;
if (type == 'r') {
how_many = tape_val & JSON_VALUE_MASK;
} else {
// Error: no starting root node?
return false;
}
os << "\t// pointing to " << how_many << " (right after last node)\n";
uint64_t payload;
for (; tape_idx < how_many; tape_idx++) {
os << tape_idx << " : ";
tape_val = tape[tape_idx];
payload = tape_val & JSON_VALUE_MASK;
type = (tape_val >> 56);
switch (type) {
case '"': // we have a string
os << "string \"";
memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t));
internal::print_with_escapes(
(const unsigned char *)(string_buf.get() + payload + sizeof(uint32_t)),
os,
string_length);
os << '"';
os << '\n';
break;
case 'l': // we have a long int
if (tape_idx + 1 >= how_many) {
return false;
}
os << "integer " << static_cast<int64_t>(tape[++tape_idx]) << "\n";
break;
case 'u': // we have a long uint
if (tape_idx + 1 >= how_many) {
return false;
}
os << "unsigned integer " << tape[++tape_idx] << "\n";
break;
case 'd': // we have a double
os << "float ";
if (tape_idx + 1 >= how_many) {
return false;
}
double answer;
memcpy(&answer, &tape[++tape_idx], sizeof(answer));
os << answer << '\n';
break;
case 'n': // we have a null
os << "null\n";
break;
case 't': // we have a true
os << "true\n";
break;
case 'f': // we have a false
os << "false\n";
break;
case '{': // we have an object
os << "{\t// pointing to next tape location " << payload
<< " (first node after the scope) \n";
break;
case '}': // we end an object
os << "}\t// pointing to previous tape location " << payload
<< " (start of the scope) \n";
break;
case '[': // we start an array
os << "[\t// pointing to next tape location " << payload
<< " (first node after the scope) \n";
break;
case ']': // we end an array
os << "]\t// pointing to previous tape location " << payload
<< " (start of the scope) \n";
break;
case 'r': // we start and end with the root node
// should we be hitting the root node?
return false;
default:
return false;
}
}
tape_val = tape[tape_idx];
payload = tape_val & JSON_VALUE_MASK;
type = (tape_val >> 56);
os << tape_idx << " : " << type << "\t// pointing to " << payload
<< " (start root)\n";
return true;
}
//
// document::doc_ref_result inline implementation
//
inline document::doc_ref_result::doc_ref_result(document &_doc, error_code _error) noexcept : doc(_doc), error(_error) { }
inline document::doc_ref_result::operator document&() noexcept(false) {
if (error) {
throw invalid_json(error);
}
return doc;
}
inline const std::string &document::doc_ref_result::get_error_message() const noexcept {
return error_message(error);
}
//
// document::doc_result inline implementation
//
inline document::doc_result::doc_result(document &&_doc, error_code _error) noexcept : doc(std::move(_doc)), error(_error) { }
inline document::doc_result::doc_result(document &&_doc) noexcept : doc(std::move(_doc)), error(SUCCESS) { }
inline document::doc_result::doc_result(error_code _error) noexcept : doc(), error(_error) { }
inline document::doc_result::operator document() noexcept(false) {
if (error) {
throw invalid_json(error);
}
return std::move(doc);
}
inline const std::string &document::doc_result::get_error_message() const noexcept {
return error_message(error);
}
//
// document::parser inline implementation
//
inline bool document::parser::is_valid() const noexcept { return valid; }
inline int document::parser::get_error_code() const noexcept { return error; }
inline std::string document::parser::get_error_message() const noexcept { return error_message(error); }
inline bool document::parser::print_json(std::ostream &os) const noexcept {
return is_valid() ? doc.print_json(os) : false;
}
inline bool document::parser::dump_raw_tape(std::ostream &os) const noexcept {
return is_valid() ? doc.dump_raw_tape(os) : false;
}
inline const document &document::parser::get_document() const noexcept(false) {
if (!is_valid()) {
throw invalid_json(error);
}
return doc;
}
inline document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept {
error_code code = init_parse(len);
if (code) { return document::doc_ref_result(doc, code); }
if (realloc_if_needed) {
const uint8_t *tmp_buf = buf;
buf = (uint8_t *)allocate_padded_buffer(len);
if (buf == nullptr)
return document::doc_ref_result(doc, MEMALLOC);
memcpy((void *)buf, tmp_buf, len);
}
code = simdjson::active_implementation->parse(buf, len, *this);
// We're indicating validity via the doc_ref_result, so set the parse state back to invalid
valid = false;
error = UNINITIALIZED;
if (realloc_if_needed) {
aligned_free((void *)buf); // must free before we exit
}
return document::doc_ref_result(doc, code);
}
really_inline document::doc_ref_result document::parser::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept {
return parse((const uint8_t *)buf, len, realloc_if_needed);
}
really_inline document::doc_ref_result document::parser::parse(const std::string &s) noexcept {
return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
}
really_inline document::doc_ref_result document::parser::parse(const padded_string &s) noexcept {
return parse(s.data(), s.length(), false);
}
really_inline size_t document::parser::capacity() const noexcept {
return _capacity;
}
really_inline size_t document::parser::max_depth() const noexcept {
return _max_depth;
}
WARN_UNUSED inline bool document::parser::allocate_capacity(size_t capacity, size_t max_depth) {
return set_capacity(capacity) && set_max_depth(max_depth);
}
WARN_UNUSED
inline bool document::parser::set_capacity(size_t capacity) {
if (_capacity == capacity) {
return true;
}
// Set capacity to 0 until we finish, in case there's an error
_capacity = 0;
//
// Reallocate the document
//
if (!doc.set_capacity(capacity)) {
return false;
}
//
// Don't allocate 0 bytes, just return.
//
if (capacity == 0) {
structural_indexes.reset();
return true;
}
//
// Initialize stage 1 output
//
uint32_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7;
structural_indexes.reset( new (std::nothrow) uint32_t[max_structures]); // TODO realloc
if (!structural_indexes) {
return false;
}
_capacity = capacity;
return true;
}
WARN_UNUSED inline bool document::parser::set_max_depth(size_t max_depth) {
_max_depth = 0;
if (max_depth == 0) {
ret_address.reset();
containing_scope_offset.reset();
return true;
}
//
// Initialize stage 2 state
//
containing_scope_offset.reset(new (std::nothrow) uint32_t[max_depth]); // TODO realloc
#ifdef SIMDJSON_USE_COMPUTED_GOTO
ret_address.reset(new (std::nothrow) void *[max_depth]);
#else
ret_address.reset(new (std::nothrow) char[max_depth]);
#endif
if (!ret_address || !containing_scope_offset) {
// Could not allocate memory
return false;
}
_max_depth = max_depth;
return true;
}
WARN_UNUSED
inline error_code document::parser::init_parse(size_t len) noexcept {
if (len > capacity()) {
return error = CAPACITY;
}
// If the last doc was taken, we need to allocate a new one
if (!doc.tape) {
if (!doc.set_capacity(len)) {
return error = MEMALLOC;
}
}
return SUCCESS;
}
//
// document::tape_ref inline implementation
//
really_inline document::tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {}
really_inline document::tape_ref::tape_ref(const document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {}
inline size_t document::tape_ref::after_element() const noexcept {
switch (type()) {
case tape_type::START_ARRAY:
case tape_type::START_OBJECT:
return tape_value();
case tape_type::UINT64:
case tape_type::INT64:
case tape_type::DOUBLE:
return json_index + 2;
default:
return json_index + 1;
}
}
really_inline document::tape_type document::tape_ref::type() const noexcept {
return static_cast<tape_type>(doc->tape[json_index] >> 56);
}
really_inline uint64_t document::tape_ref::tape_value() const noexcept {
return doc->tape[json_index] & JSON_VALUE_MASK;
}
template<typename T>
really_inline T document::tape_ref::next_tape_value() const noexcept {
static_assert(sizeof(T) == sizeof(uint64_t));
return *reinterpret_cast<const T*>(&doc->tape[json_index + 1]);
}
//
// document::array inline implementation
//
really_inline document::array::array() noexcept : tape_ref() {}
really_inline document::array::array(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) {}
inline document::array::iterator document::array::begin() const noexcept {
return iterator(doc, json_index + 1);
}
inline document::array::iterator document::array::end() const noexcept {
return iterator(doc, after_element() - 1);
}
//
// document::array::iterator inline implementation
//
really_inline document::array::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { }
inline document::element document::array::iterator::operator*() const noexcept {
return element(doc, json_index);
}
inline bool document::array::iterator::operator!=(const document::array::iterator& other) const noexcept {
return json_index != other.json_index;
}
inline void document::array::iterator::operator++() noexcept {
json_index = after_element();
}
//
// document::object inline implementation
//
really_inline document::object::object() noexcept : tape_ref() {}
really_inline document::object::object(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { };
inline document::object::iterator document::object::begin() const noexcept {
return iterator(doc, json_index + 1);
}
inline document::object::iterator document::object::end() const noexcept {
return iterator(doc, after_element() - 1);
}
inline document::element_result<document::element> document::object::operator[](const std::string_view &key) const noexcept {
iterator end_field = end();
for (iterator field = begin(); field != end_field; ++field) {
if (key == field.key()) {
return field.value();
}
}
return NO_SUCH_FIELD;
}
inline document::element_result<document::element> document::object::operator[](const char *key) const noexcept {
iterator end_field = end();
for (iterator field = begin(); field != end_field; ++field) {
if (!strcmp(key, field.key_c_str())) {
return field.value();
}
}
return NO_SUCH_FIELD;
}
//
// document::object::iterator inline implementation
//
really_inline document::object::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { }
inline const document::key_value_pair document::object::iterator::operator*() const noexcept {
return key_value_pair(key(), value());
}
inline bool document::object::iterator::operator!=(const document::object::iterator& other) const noexcept {
return json_index != other.json_index;
}
inline void document::object::iterator::operator++() noexcept {
json_index++;
json_index = after_element();
}
inline std::string_view document::object::iterator::key() const noexcept {
size_t string_buf_index = tape_value();
uint32_t len;
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
return std::string_view(
reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]),
len
);
}
inline const char* document::object::iterator::key_c_str() const noexcept {
return reinterpret_cast<const char *>(&doc->string_buf[tape_value() + sizeof(uint32_t)]);
}
inline document::element document::object::iterator::value() const noexcept {
return element(doc, json_index + 1);
}
//
// document::key_value_pair inline implementation
//
inline document::key_value_pair::key_value_pair(std::string_view _key, document::element _value) noexcept :
key(_key), value(_value) {}
//
// document::element inline implementation
//
really_inline document::element::element() noexcept : tape_ref() {}
really_inline document::element::element(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { }
really_inline bool document::element::is_null() const noexcept {
return type() == tape_type::NULL_VALUE;
}
really_inline bool document::element::is_bool() const noexcept {
return type() == tape_type::TRUE_VALUE || type() == tape_type::FALSE_VALUE;
}
really_inline bool document::element::is_number() const noexcept {
return type() == tape_type::UINT64 || type() == tape_type::INT64 || type() == tape_type::DOUBLE;
}
really_inline bool document::element::is_integer() const noexcept {
return type() == tape_type::UINT64 || type() == tape_type::INT64;
}
really_inline bool document::element::is_string() const noexcept {
return type() == tape_type::STRING;
}
really_inline bool document::element::is_array() const noexcept {
return type() == tape_type::START_ARRAY;
}
really_inline bool document::element::is_object() const noexcept {
return type() == tape_type::START_OBJECT;
}
inline document::element::operator bool() const noexcept(false) { return as_bool(); }
inline document::element::operator const char*() const noexcept(false) { return as_c_str(); }
inline document::element::operator std::string_view() const noexcept(false) { return as_string(); }
inline document::element::operator uint64_t() const noexcept(false) { return as_uint64_t(); }
inline document::element::operator int64_t() const noexcept(false) { return as_int64_t(); }
inline document::element::operator double() const noexcept(false) { return as_double(); }
inline document::element::operator document::array() const noexcept(false) { return as_array(); }
inline document::element::operator document::object() const noexcept(false) { return as_object(); }
inline document::element_result<bool> document::element::as_bool() const noexcept {
switch (type()) {
case tape_type::TRUE_VALUE:
return true;
case tape_type::FALSE_VALUE:
return false;
default:
return INCORRECT_TYPE;
}
}
inline document::element_result<const char *> document::element::as_c_str() const noexcept {
switch (type()) {
case tape_type::STRING: {
size_t string_buf_index = tape_value();
return reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]);
}
default:
return INCORRECT_TYPE;
}
}
inline document::element_result<std::string_view> document::element::as_string() const noexcept {
switch (type()) {
case tape_type::STRING: {
size_t string_buf_index = tape_value();
uint32_t len;
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
return std::string_view(
reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]),
len
);
}
default:
return INCORRECT_TYPE;
}
}
inline document::element_result<uint64_t> document::element::as_uint64_t() const noexcept {
switch (type()) {
case tape_type::UINT64:
return next_tape_value<uint64_t>();
case tape_type::INT64: {
int64_t result = next_tape_value<int64_t>();
if (result < 0) {
return NUMBER_OUT_OF_RANGE;
}
return static_cast<uint64_t>(result);
}
default:
return INCORRECT_TYPE;
}
}
inline document::element_result<int64_t> document::element::as_int64_t() const noexcept {
switch (type()) {
case tape_type::UINT64: {
uint64_t result = next_tape_value<uint64_t>();
// Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std
if (result > (std::numeric_limits<uint64_t>::max)()) {
return NUMBER_OUT_OF_RANGE;
}
return static_cast<int64_t>(result);
}
case tape_type::INT64:
return next_tape_value<int64_t>();
default:
std::cout << "Incorrect " << json_index << " = " << char(type()) << std::endl;
return INCORRECT_TYPE;
}
}
inline document::element_result<double> document::element::as_double() const noexcept {
switch (type()) {
case tape_type::UINT64:
return next_tape_value<uint64_t>();
case tape_type::INT64: {
return next_tape_value<int64_t>();
int64_t result = tape_value();
if (result < 0) {
return NUMBER_OUT_OF_RANGE;
}
return result;
}
case tape_type::DOUBLE:
return next_tape_value<double>();
default:
return INCORRECT_TYPE;
}
}
inline document::element_result<document::array> document::element::as_array() const noexcept {
switch (type()) {
case tape_type::START_ARRAY:
return array(doc, json_index);
default:
return INCORRECT_TYPE;
}
}
inline document::element_result<document::object> document::element::as_object() const noexcept {
switch (type()) {
case tape_type::START_OBJECT:
return object(doc, json_index);
default:
return INCORRECT_TYPE;
}
}
inline document::element_result<document::element> document::element::operator[](const std::string_view &key) const noexcept {
auto [obj, error] = as_object();
if (error) { return error; }
return obj[key];
}
inline document::element_result<document::element> document::element::operator[](const char *key) const noexcept {
auto [obj, error] = as_object();
if (error) { return error; }
return obj[key];
}
} // namespace simdjson
#endif // SIMDJSON_INLINE_DOCUMENT_H
/* end file include/simdjson/inline/document.h */
/* begin file include/simdjson/inline/document_iterator.h */
#ifndef SIMDJSON_INLINE_DOCUMENT_ITERATOR_H
#define SIMDJSON_INLINE_DOCUMENT_ITERATOR_H
namespace simdjson {
// Because of template weirdness, the actual class definition is inline in the document class
template <size_t max_depth>
WARN_UNUSED bool document_iterator<max_depth>::is_ok() const {
return location < tape_length;
}
// useful for debuging purposes
template <size_t max_depth>
size_t document_iterator<max_depth>::get_tape_location() const {
return location;
}
// useful for debuging purposes
template <size_t max_depth>
size_t document_iterator<max_depth>::get_tape_length() const {
return tape_length;
}
// returns the current depth (start at 1 with 0 reserved for the fictitious root
// node)
template <size_t max_depth>
size_t document_iterator<max_depth>::get_depth() const {
return depth;
}
// A scope is a series of nodes at the same depth, typically it is either an
// object ({) or an array ([). The root node has type 'r'.
template <size_t max_depth>
uint8_t document_iterator<max_depth>::get_scope_type() const {
return depth_index[depth].scope_type;
}
template <size_t max_depth>
bool document_iterator<max_depth>::move_forward() {
if (location + 1 >= tape_length) {
return false; // we are at the end!
}
if ((current_type == '[') || (current_type == '{')) {
// We are entering a new scope
depth++;
assert(depth < max_depth);
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
} else if ((current_type == ']') || (current_type == '}')) {
// Leaving a scope.
depth--;
} else if (is_number()) {
// these types use 2 locations on the tape, not just one.
location += 1;
}
location += 1;
current_val = doc.tape[location];
current_type = (current_val >> 56);
return true;
}
template <size_t max_depth>
void document_iterator<max_depth>::move_to_value() {
// assume that we are on a key, so move by 1.
location += 1;
current_val = doc.tape[location];
current_type = (current_val >> 56);
}
template <size_t max_depth>
bool document_iterator<max_depth>::move_to_key(const char *key) {
if (down()) {
do {
const bool right_key = (strcmp(get_string(), key) == 0);
move_to_value();
if (right_key) {
return true;
}
} while (next());
up();
}
return false;
}
template <size_t max_depth>
bool document_iterator<max_depth>::move_to_key_insensitive(
const char *key) {
if (down()) {
do {
const bool right_key = (simdjson_strcasecmp(get_string(), key) == 0);
move_to_value();
if (right_key) {
return true;
}
} while (next());
up();
}
return false;
}
template <size_t max_depth>
bool document_iterator<max_depth>::move_to_key(const char *key,
uint32_t length) {
if (down()) {
do {
bool right_key = ((get_string_length() == length) &&
(memcmp(get_string(), key, length) == 0));
move_to_value();
if (right_key) {
return true;
}
} while (next());
up();
}
return false;
}
template <size_t max_depth>
bool document_iterator<max_depth>::move_to_index(uint32_t index) {
if (down()) {
uint32_t i = 0;
for (; i < index; i++) {
if (!next()) {
break;
}
}
if (i == index) {
return true;
}
up();
}
return false;
}
template <size_t max_depth> bool document_iterator<max_depth>::prev() {
size_t target_location = location;
to_start_scope();
size_t npos = location;
if (target_location == npos) {
return false; // we were already at the start
}
size_t oldnpos;
// we have that npos < target_location here
do {
oldnpos = npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = (current_val & JSON_VALUE_MASK);
} else {
npos = npos + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
}
} while (npos < target_location);
location = oldnpos;
current_val = doc.tape[location];
current_type = current_val >> 56;
return true;
}
template <size_t max_depth> bool document_iterator<max_depth>::up() {
if (depth == 1) {
return false; // don't allow moving back to root
}
to_start_scope();
// next we just move to the previous value
depth--;
location -= 1;
current_val = doc.tape[location];
current_type = (current_val >> 56);
return true;
}
template <size_t max_depth> bool document_iterator<max_depth>::down() {
if (location + 1 >= tape_length) {
return false;
}
if ((current_type == '[') || (current_type == '{')) {
size_t npos = (current_val & JSON_VALUE_MASK);
if (npos == location + 2) {
return false; // we have an empty scope
}
depth++;
assert(depth < max_depth);
location = location + 1;
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
current_val = doc.tape[location];
current_type = (current_val >> 56);
return true;
}
return false;
}
template <size_t max_depth>
void document_iterator<max_depth>::to_start_scope() {
location = depth_index[depth].start_of_scope;
current_val = doc.tape[location];
current_type = (current_val >> 56);
}
template <size_t max_depth> bool document_iterator<max_depth>::next() {
size_t npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = (current_val & JSON_VALUE_MASK);
} else {
npos = location + (is_number() ? 2 : 1);
}
uint64_t next_val = doc.tape[npos];
uint8_t next_type = (next_val >> 56);
if ((next_type == ']') || (next_type == '}')) {
return false; // we reached the end of the scope
}
location = npos;
current_val = next_val;
current_type = next_type;
return true;
}
template <size_t max_depth>
document_iterator<max_depth>::document_iterator(const document &doc_) noexcept
: doc(doc_), depth(0), location(0), tape_length(0) {
depth_index[0].start_of_scope = location;
current_val = doc.tape[location++];
current_type = (current_val >> 56);
depth_index[0].scope_type = current_type;
tape_length = current_val & JSON_VALUE_MASK;
if (location < tape_length) {
// If we make it here, then depth_capacity must >=2, but the compiler
// may not know this.
current_val = doc.tape[location];
current_type = (current_val >> 56);
depth++;
assert(depth < max_depth);
depth_index[depth].start_of_scope = location;
depth_index[depth].scope_type = current_type;
}
}
template <size_t max_depth>
document_iterator<max_depth>::document_iterator(const document::parser &parser)
: document_iterator(parser.get_document()) {}
template <size_t max_depth>
document_iterator<max_depth>::document_iterator(
const document_iterator &o) noexcept
: doc(o.doc), depth(o.depth), location(o.location),
tape_length(o.tape_length), current_type(o.current_type),
current_val(o.current_val) {
memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0]));
}
template <size_t max_depth>
document_iterator<max_depth> &document_iterator<max_depth>::
operator=(const document_iterator &o) noexcept {
doc = o.doc;
depth = o.depth;
location = o.location;
tape_length = o.tape_length;
current_type = o.current_type;
current_val = o.current_val;
memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0]));
return *this;
}
template <size_t max_depth>
bool document_iterator<max_depth>::print(std::ostream &os, bool escape_strings) const {
if (!is_ok()) {
return false;
}
switch (current_type) {
case '"': // we have a string
os << '"';
if (escape_strings) {
internal::print_with_escapes(get_string(), os, get_string_length());
} else {
// was: os << get_string();, but given that we can include null chars, we
// have to do something crazier:
std::copy(get_string(), get_string() + get_string_length(), std::ostream_iterator<char>(os));
}
os << '"';
break;
case 'l': // we have a long int
os << get_integer();
break;
case 'u':
os << get_unsigned_integer();
break;
case 'd':
os << get_double();
break;
case 'n': // we have a null
os << "null";
break;
case 't': // we have a true
os << "true";
break;
case 'f': // we have a false
os << "false";
break;
case '{': // we have an object
case '}': // we end an object
case '[': // we start an array
case ']': // we end an array
os << static_cast<char>(current_type);
break;
default:
return false;
}
return true;
}
template <size_t max_depth>
bool document_iterator<max_depth>::move_to(const char *pointer,
uint32_t length) {
char *new_pointer = nullptr;
if (pointer[0] == '#') {
// Converting fragment representation to string representation
new_pointer = new char[length];
uint32_t new_length = 0;
for (uint32_t i = 1; i < length; i++) {
if (pointer[i] == '%' && pointer[i + 1] == 'x') {
try {
int fragment =
std::stoi(std::string(&pointer[i + 2], 2), nullptr, 16);
if (fragment == '\\' || fragment == '"' || (fragment <= 0x1F)) {
// escaping the character
new_pointer[new_length] = '\\';
new_length++;
}
new_pointer[new_length] = fragment;
i += 3;
} catch (std::invalid_argument &) {
delete[] new_pointer;
return false; // the fragment is invalid
}
} else {
new_pointer[new_length] = pointer[i];
}
new_length++;
}
length = new_length;
pointer = new_pointer;
}
// saving the current state
size_t depth_s = depth;
size_t location_s = location;
uint8_t current_type_s = current_type;
uint64_t current_val_s = current_val;
rewind(); // The json pointer is used from the root of the document.
bool found = relative_move_to(pointer, length);
delete[] new_pointer;
if (!found) {
// since the pointer has found nothing, we get back to the original
// position.
depth = depth_s;
location = location_s;
current_type = current_type_s;
current_val = current_val_s;
}
return found;
}
template <size_t max_depth>
bool document_iterator<max_depth>::relative_move_to(const char *pointer,
uint32_t length) {
if (length == 0) {
// returns the whole document
return true;
}
if (pointer[0] != '/') {
// '/' must be the first character
return false;
}
// finding the key in an object or the index in an array
std::string key_or_index;
uint32_t offset = 1;
// checking for the "-" case
if (is_array() && pointer[1] == '-') {
if (length != 2) {
// the pointer must be exactly "/-"
// there can't be anything more after '-' as an index
return false;
}
key_or_index = '-';
offset = length; // will skip the loop coming right after
}
// We either transform the first reference token to a valid json key
// or we make sure it is a valid index in an array.
for (; offset < length; offset++) {
if (pointer[offset] == '/') {
// beginning of the next key or index
break;
}
if (is_array() && (pointer[offset] < '0' || pointer[offset] > '9')) {
// the index of an array must be an integer
// we also make sure std::stoi won't discard whitespaces later
return false;
}
if (pointer[offset] == '~') {
// "~1" represents "/"
if (pointer[offset + 1] == '1') {
key_or_index += '/';
offset++;
continue;
}
// "~0" represents "~"
if (pointer[offset + 1] == '0') {
key_or_index += '~';
offset++;
continue;
}
}
if (pointer[offset] == '\\') {
if (pointer[offset + 1] == '\\' || pointer[offset + 1] == '"' ||
(pointer[offset + 1] <= 0x1F)) {
key_or_index += pointer[offset + 1];
offset++;
continue;
}
return false; // invalid escaped character
}
if (pointer[offset] == '\"') {
// unescaped quote character. this is an invalid case.
// lets do nothing and assume most pointers will be valid.
// it won't find any corresponding json key anyway.
// return false;
}
key_or_index += pointer[offset];
}
bool found = false;
if (is_object()) {
if (move_to_key(key_or_index.c_str(), key_or_index.length())) {
found = relative_move_to(pointer + offset, length - offset);
}
} else if (is_array()) {
if (key_or_index == "-") { // handling "-" case first
if (down()) {
while (next())
; // moving to the end of the array
// moving to the nonexistent value right after...
size_t npos;
if ((current_type == '[') || (current_type == '{')) {
// we need to jump
npos = (current_val & JSON_VALUE_MASK);
} else {
npos =
location + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
}
location = npos;
current_val = doc.tape[npos];
current_type = (current_val >> 56);
return true; // how could it fail ?
}
} else { // regular numeric index
// The index can't have a leading '0'
if (key_or_index[0] == '0' && key_or_index.length() > 1) {
return false;
}
// it cannot be empty
if (key_or_index.length() == 0) {
return false;
}
// we already checked the index contains only valid digits
uint32_t index = std::stoi(key_or_index);
if (move_to_index(index)) {
found = relative_move_to(pointer + offset, length - offset);
}
}
}
return found;
}
} // namespace simdjson
#endif // SIMDJSON_INLINE_DOCUMENT_ITERATOR_H
/* end file include/simdjson/inline/document_iterator.h */
/* begin file include/simdjson/inline/jsonstream.h */
#ifndef SIMDJSON_INLINE_JSONSTREAM_H
#define SIMDJSON_INLINE_JSONSTREAM_H
#include <algorithm>
#include <limits>
#include <stdexcept>
#include <thread>
namespace simdjson::internal {
/* This algorithm is used to quickly identify the buffer position of
* the last JSON document inside the current batch.
*
* It does its work by finding the last pair of structural characters
* that represent the end followed by the start of a document.
*
* Simply put, we iterate over the structural characters, starting from
* the end. We consider that we found the end of a JSON document when the
* first element of the pair is NOT one of these characters: '{' '[' ';' ','
* and when the second element is NOT one of these characters: '}' '}' ';' ','.
*
* This simple comparison works most of the time, but it does not cover cases
* where the batch's structural indexes contain a perfect amount of documents.
* In such a case, we do not have access to the structural index which follows
* the last document, therefore, we do not have access to the second element in
* the pair, and means that we cannot identify the last document. To fix this
* issue, we keep a count of the open and closed curly/square braces we found
* while searching for the pair. When we find a pair AND the count of open and
* closed curly/square braces is the same, we know that we just passed a
* complete
* document, therefore the last json buffer location is the end of the batch
* */
inline size_t find_last_json_buf_idx(const uint8_t *buf, size_t size, const document::parser &parser) {
// this function can be generally useful
if (parser.n_structural_indexes == 0)
return 0;
auto last_i = parser.n_structural_indexes - 1;
if (parser.structural_indexes[last_i] == size) {
if (last_i == 0)
return 0;
last_i = parser.n_structural_indexes - 2;
}
auto arr_cnt = 0;
auto obj_cnt = 0;
for (auto i = last_i; i > 0; i--) {
auto idxb = parser.structural_indexes[i];
switch (buf[idxb]) {
case ':':
case ',':
continue;
case '}':
obj_cnt--;
continue;
case ']':
arr_cnt--;
continue;
case '{':
obj_cnt++;
break;
case '[':
arr_cnt++;
break;
}
auto idxa = parser.structural_indexes[i - 1];
switch (buf[idxa]) {
case '{':
case '[':
case ':':
case ',':
continue;
}
if (!arr_cnt && !obj_cnt) {
return last_i + 1;
}
return i;
}
return 0;
}
// returns true if the provided byte value is an ASCII character
static inline bool is_ascii(char c) {
return ((unsigned char)c) <= 127;
}
// if the string ends with UTF-8 values, backtrack
// up to the first ASCII character. May return 0.
static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) {
while ((len > 0) and (not is_ascii(c[len - 1]))) {
len--;
}
return len;
}
} // namespace simdjson::internal
namespace simdjson {
template <class string_container>
JsonStream<string_container>::JsonStream(const string_container &s,
size_t batchSize)
: str(s), _batch_size(batchSize) {
}
template <class string_container> JsonStream<string_container>::~JsonStream() {
#ifdef SIMDJSON_THREADS_ENABLED
if (stage_1_thread.joinable()) {
stage_1_thread.join();
}
#endif
}
#ifdef SIMDJSON_THREADS_ENABLED
// threaded version of json_parse
// todo: simplify this code further
template <class string_container>
int JsonStream<string_container>::json_parse(document::parser &parser) {
if (unlikely(parser.capacity() == 0)) {
const bool allocok = parser.allocate_capacity(_batch_size);
if (!allocok) {
return parser.error = simdjson::MEMALLOC;
}
} else if (unlikely(parser.capacity() < _batch_size)) {
return parser.error = simdjson::CAPACITY;
}
if (unlikely(parser_thread.capacity() < _batch_size)) {
const bool allocok_thread = parser_thread.allocate_capacity(_batch_size);
if (!allocok_thread) {
return parser.error = simdjson::MEMALLOC;
}
}
if (unlikely(load_next_batch)) {
// First time loading
if (!stage_1_thread.joinable()) {
_batch_size = (std::min)(_batch_size, remaining());
_batch_size = internal::trimmed_length_safe_utf8((const char *)buf(), _batch_size);
if (_batch_size == 0) {
return parser.error = simdjson::UTF8_ERROR;
}
auto stage1_is_ok = error_code(simdjson::active_implementation->stage1(buf(), _batch_size, parser, true));
if (stage1_is_ok != simdjson::SUCCESS) {
return parser.error = stage1_is_ok;
}
size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
if (last_index == 0) {
if (parser.n_structural_indexes == 0) {
return parser.error = simdjson::EMPTY;
}
} else {
parser.n_structural_indexes = last_index + 1;
}
}
// the second thread is running or done.
else {
stage_1_thread.join();
if (stage1_is_ok_thread != simdjson::SUCCESS) {
return parser.error = stage1_is_ok_thread;
}
std::swap(parser.structural_indexes, parser_thread.structural_indexes);
parser.n_structural_indexes = parser_thread.n_structural_indexes;
advance(last_json_buffer_loc);
n_bytes_parsed += last_json_buffer_loc;
}
// let us decide whether we will start a new thread
if (remaining() - _batch_size > 0) {
last_json_buffer_loc =
parser.structural_indexes[internal::find_last_json_buf_idx(buf(), _batch_size, parser)];
_batch_size = (std::min)(_batch_size, remaining() - last_json_buffer_loc);
if (_batch_size > 0) {
_batch_size = internal::trimmed_length_safe_utf8(
(const char *)(buf() + last_json_buffer_loc), _batch_size);
if (_batch_size == 0) {
return parser.error = simdjson::UTF8_ERROR;
}
// let us capture read-only variables
const uint8_t *const b = buf() + last_json_buffer_loc;
const size_t bs = _batch_size;
// we call the thread on a lambda that will update
// this->stage1_is_ok_thread
// there is only one thread that may write to this value
stage_1_thread = std::thread([this, b, bs] {
this->stage1_is_ok_thread = error_code(simdjson::active_implementation->stage1(b, bs, this->parser_thread, true));
});
}
}
next_json = 0;
load_next_batch = false;
} // load_next_batch
int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json);
if (res == simdjson::SUCCESS_AND_HAS_MORE) {
n_parsed_docs++;
current_buffer_loc = parser.structural_indexes[next_json];
load_next_batch = (current_buffer_loc == last_json_buffer_loc);
} else if (res == simdjson::SUCCESS) {
n_parsed_docs++;
if (remaining() > _batch_size) {
current_buffer_loc = parser.structural_indexes[next_json - 1];
load_next_batch = true;
res = simdjson::SUCCESS_AND_HAS_MORE;
}
}
return res;
}
#else // SIMDJSON_THREADS_ENABLED
// single-threaded version of json_parse
template <class string_container>
int JsonStream<string_container>::json_parse(document::parser &parser) {
if (unlikely(parser.capacity() == 0)) {
const bool allocok = parser.allocate_capacity(_batch_size);
if (!allocok) {
parser.valid = false;
return parser.error = MEMALLOC;
}
} else if (unlikely(parser.capacity() < _batch_size)) {
parser.valid = false;
return parser.error = CAPACITY;
}
if (unlikely(load_next_batch)) {
advance(current_buffer_loc);
n_bytes_parsed += current_buffer_loc;
_batch_size = (std::min)(_batch_size, remaining());
_batch_size = internal::trimmed_length_safe_utf8((const char *)buf(), _batch_size);
auto stage1_is_ok = (error_code)simdjson::active_implementation->stage1(buf(), _batch_size, parser, true);
if (stage1_is_ok != simdjson::SUCCESS) {
parser.valid = false;
return parser.error = stage1_is_ok;
}
size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser);
if (last_index == 0) {
if (parser.n_structural_indexes == 0) {
parser.valid = false;
return parser.error = EMPTY;
}
} else {
parser.n_structural_indexes = last_index + 1;
}
load_next_batch = false;
} // load_next_batch
int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json);
if (likely(res == simdjson::SUCCESS_AND_HAS_MORE)) {
n_parsed_docs++;
current_buffer_loc = parser.structural_indexes[next_json];
} else if (res == simdjson::SUCCESS) {
n_parsed_docs++;
if (remaining() > _batch_size) {
current_buffer_loc = parser.structural_indexes[next_json - 1];
next_json = 1;
load_next_batch = true;
res = simdjson::SUCCESS_AND_HAS_MORE;
}
} else {
printf("E\n");
}
return res;
}
#endif // SIMDJSON_THREADS_ENABLED
} // end of namespace simdjson
#endif // SIMDJSON_INLINE_JSONSTREAM_H
/* end file include/simdjson/inline/jsonstream.h */
#endif // SIMDJSON_H
/* end file include/simdjson/inline/jsonstream.h */
Reference in New Issue View Git Blame Copy Permalink