jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Make all values use same underlying iterator

This commit is contained in:
John Keiser 2020-11-30 14:30:00 -08:00
parent 1303a88769
commit c89647af9e
26 changed files with 1279 additions and 1068 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
benchmark/bench_ondemand.cpp
View File

@ -8,15 +8,15 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
SIMDJSON_POP_DISABLE_WARNINGS
#include "partial_tweets/ondemand.h"
#include "partial_tweets/iter.h"
// #include "partial_tweets/iter.h"
#include "partial_tweets/dom.h"
#include "largerandom/ondemand.h"
#include "largerandom/iter.h"
// #include "largerandom/iter.h"
#include "largerandom/dom.h"
#include "kostya/ondemand.h"
#include "kostya/iter.h"
// #include "kostya/iter.h"
#include "kostya/dom.h"
#include "distinctuserid/ondemand.h"

1
include/simdjson/generic/ondemand-inl.h
View File

@ -2,6 +2,7 @@
#include "simdjson/generic/ondemand/raw_json_string-inl.h"
#include "simdjson/generic/ondemand/token_iterator-inl.h"
#include "simdjson/generic/ondemand/json_iterator-inl.h"
#include "simdjson/generic/ondemand/value_iterator-inl.h"
#include "simdjson/generic/ondemand/array_iterator-inl.h"
#include "simdjson/generic/ondemand/object_iterator-inl.h"
#include "simdjson/generic/ondemand/array-inl.h"

2
include/simdjson/generic/ondemand.h
View File

@ -6,6 +6,7 @@ namespace SIMDJSON_IMPLEMENTATION {
* Designed for maximum speed and a lower memory profile.
*/
namespace ondemand {
using depth_t = int32_t;
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
@ -14,6 +15,7 @@ namespace ondemand {
#include "simdjson/generic/ondemand/raw_json_string.h"
#include "simdjson/generic/ondemand/token_iterator.h"
#include "simdjson/generic/ondemand/json_iterator.h"
#include "simdjson/generic/ondemand/value_iterator.h"
#include "simdjson/generic/ondemand/array_iterator.h"
#include "simdjson/generic/ondemand/object_iterator.h"
#include "simdjson/generic/ondemand/array.h"

63
include/simdjson/generic/ondemand/array-inl.h
View File

@ -40,57 +40,30 @@ namespace ondemand {
// error == SUCCESS.
//
simdjson_really_inline array::array(json_iterator_ref &&_iter) noexcept
: iter{std::forward<json_iterator_ref>(_iter)}
simdjson_really_inline array::array(const value_iterator &_iter) noexcept
: iter{_iter}
{
}
simdjson_really_inline simdjson_result<array> array::start(json_iterator_ref &&iter) noexcept {
bool has_value;
SIMDJSON_TRY( iter->start_array().get(has_value) );
if (has_value) { iter.started_container(); } else { iter.abandon(); }
return array(std::forward<json_iterator_ref>(iter));
simdjson_really_inline simdjson_result<array> array::start(value_iterator &iter) noexcept {
simdjson_unused bool has_value;
SIMDJSON_TRY( iter.start_array().get(has_value) );
return array(iter);
}
simdjson_really_inline simdjson_result<array> array::start(const uint8_t *json, json_iterator_ref &&iter) noexcept {
bool has_value;
SIMDJSON_TRY( iter->start_array(json).get(has_value) );
if (has_value) { iter.started_container(); } else { iter.abandon(); }
return array(std::forward<json_iterator_ref>(iter));
simdjson_really_inline simdjson_result<array> array::try_start(value_iterator &iter) noexcept {
simdjson_unused bool has_value;
SIMDJSON_TRY( iter.try_start_array().get(has_value) );
return array(iter);
}
simdjson_really_inline array array::started(json_iterator_ref &&iter) noexcept {
if (iter->started_array()) { iter.started_container(); } else { iter.abandon(); }
return array(std::forward<json_iterator_ref>(iter));
simdjson_really_inline array array::started(value_iterator &iter) noexcept {
simdjson_unused bool has_value = iter.started_array();
return array(iter);
}
//
// For array_iterator
//
simdjson_really_inline json_iterator &array::get_iterator() noexcept {
return *iter;
simdjson_really_inline array_iterator array::begin() noexcept {
return iter;
}
simdjson_really_inline json_iterator_ref array::borrow_iterator_child() noexcept {
return iter.borrow();
}
simdjson_really_inline bool array::is_iterator_alive() const noexcept {
return iter.is_alive();
}
simdjson_really_inline void array::start_iterator() noexcept {
iter.started_container();
}
simdjson_really_inline void array::finish_iterator() noexcept {
iter.finished_container();
}
simdjson_really_inline void array::abandon_iterator() noexcept {
iter.abandon();
}
simdjson_warn_unused simdjson_really_inline error_code array::finish_iterator_child() noexcept {
return iter.finish_child();
}
simdjson_really_inline array_iterator<array> array::begin() & noexcept {
return *this;
}
simdjson_really_inline array_iterator<array> array::end() & noexcept {
simdjson_really_inline array_iterator array::end() noexcept {
return {};
}
@ -115,11 +88,11 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>
{
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::begin() & noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::begin() noexcept {
if (error()) { return error(); }
return first.begin();
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::end() & noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::end() noexcept {
if (error()) { return error(); }
return first.end();
}

38
include/simdjson/generic/ondemand/array.h
View File

@ -28,13 +28,13 @@ public:
*
* Part of the std::iterable interface.
*/
simdjson_really_inline array_iterator<array> begin() & noexcept;
simdjson_really_inline array_iterator begin() noexcept;
/**
* Sentinel representing the end of the array.
*
* Part of the std::iterable interface.
*/
simdjson_really_inline array_iterator<array> end() & noexcept;
simdjson_really_inline array_iterator end() noexcept;
protected:
/**
@ -44,8 +44,15 @@ protected:
* resulting array.
* @error INCORRECT_TYPE if the iterator is not at [.
*/
static simdjson_really_inline simdjson_result<array> start(json_iterator_ref &&iter) noexcept;
static simdjson_really_inline simdjson_result<array> start(const uint8_t *json, json_iterator_ref &&iter) noexcept;
static simdjson_really_inline simdjson_result<array> start(value_iterator &iter) noexcept;
/**
* Begin array iteration.
*
* @param iter The iterator. Must be where the initial [ is expected. Will be *moved* into the
* resulting array.
* @error INCORRECT_TYPE if the iterator is not at [.
*/
static simdjson_really_inline simdjson_result<array> try_start(value_iterator &iter) noexcept;
/**
* Begin array iteration.
*
@ -54,7 +61,7 @@ protected:
*
* @param iter The iterator. Must be after the initial [. Will be *moved* into the resulting array.
*/
static simdjson_really_inline array started(json_iterator_ref &&iter) noexcept;
static simdjson_really_inline array started(value_iterator &iter) noexcept;
/**
* Create an array at the given Internal array creation. Call array::start() or array::started() instead of this.
@ -63,30 +70,19 @@ protected:
* == true, or past the [] with is_alive() == false if the array is empty. Will be *moved*
* into the resulting array.
*/
simdjson_really_inline array(json_iterator_ref &&iter) noexcept;
//
// For array_iterator
//
simdjson_really_inline json_iterator &get_iterator() noexcept;
simdjson_really_inline json_iterator_ref borrow_iterator_child() noexcept;
simdjson_really_inline bool is_iterator_alive() const noexcept;
simdjson_really_inline void start_iterator() noexcept;
simdjson_really_inline void finish_iterator() noexcept;
simdjson_really_inline void abandon_iterator() noexcept;
simdjson_warn_unused simdjson_really_inline error_code finish_iterator_child() noexcept;
simdjson_really_inline array(const value_iterator &iter) noexcept;
/**
* Iterator marking current position.
*
* iter.is_alive() == false indicates iteration is complete.
*/
json_iterator_ref iter{};
value_iterator iter{};
friend class value;
friend struct simdjson_result<value>;
friend struct simdjson_result<array>;
friend class array_iterator<array>;
friend class array_iterator;
};
} // namespace ondemand
@ -105,8 +101,8 @@ public:
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> &&a) noexcept = default;
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> begin() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> end() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() noexcept;
};
} // namespace simdjson

72
include/simdjson/generic/ondemand/array_iterator-inl.h
View File

@ -2,43 +2,27 @@ namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
template<typename T>
simdjson_really_inline array_iterator<T>::array_iterator(T &_iter) noexcept : iter{&_iter} {}
simdjson_really_inline array_iterator::array_iterator(const value_iterator &_iter) noexcept
: iter{_iter}
{}
template<typename T>
simdjson_really_inline simdjson_result<array_iterator<T>> array_iterator<T>::start(T &iter, const uint8_t *json) noexcept {
bool has_value;
SIMDJSON_TRY( iter.get_iterator().start_array(json).get(has_value) );
if (has_value) { iter.start_iterator(); } else { iter.abandon_iterator(); }
return array_iterator<T>(iter);
simdjson_really_inline simdjson_result<value> array_iterator::operator*() noexcept {
if (iter.error()) { iter.abandon(); return iter.error(); }
return value(iter.child());
}
template<typename T>
simdjson_really_inline simdjson_result<value> array_iterator<T>::operator*() noexcept {
error_code error = iter->get_iterator().error();
if (error) { iter->abandon_iterator(); return error; }
return value::start(iter->borrow_iterator_child());
}
template<typename T>
simdjson_really_inline bool array_iterator<T>::operator==(const array_iterator<T> &other) const noexcept {
simdjson_really_inline bool array_iterator::operator==(const array_iterator &other) const noexcept {
return !(*this != other);
}
template<typename T>
simdjson_really_inline bool array_iterator<T>::operator!=(const array_iterator<T> &) const noexcept {
return iter->is_iterator_alive();
simdjson_really_inline bool array_iterator::operator!=(const array_iterator &) const noexcept {
return iter.is_open();
}
template<typename T>
simdjson_really_inline array_iterator<T> &array_iterator<T>::operator++() noexcept {
// TODO this is a safety rail ... users should exit loops as soon as they receive an error.
// Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free.
if (!iter->is_iterator_alive()) { return *this; } // Iterator will be released if there is an error
auto error = iter->finish_iterator_child();
if (error) { return *this; }
bool has_value;
error = iter->get_iterator().has_next_element().get(has_value);
if (error) { return *this; }
if (!has_value) { iter->finish_iterator(); }
simdjson_really_inline array_iterator &array_iterator::operator++() noexcept {
error_code error;
// PERF NOTE this is a safety rail ... users should exit loops as soon as they receive an error, so we'll never get here.
// However, it does not seem to make a perf difference, so we add it out of an abundance of caution.
if ((error = iter.error()) ) { return *this; }
if ((error = iter.finish_child() )) { return *this; }
if ((error = iter.has_next_element().error() )) { return *this; }
return *this;
}
@ -48,36 +32,30 @@ simdjson_really_inline array_iterator<T> &array_iterator<T>::operator++() noexce
namespace simdjson {
template<typename T>
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::simdjson_result(
SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T> &&value
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::simdjson_result(
SIMDJSON_IMPLEMENTATION::ondemand::array_iterator &&value
) noexcept
: SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>(value))
: SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>(value))
{
}
template<typename T>
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::simdjson_result(error_code error) noexcept
: SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>({}, error)
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::simdjson_result(error_code error) noexcept
: SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>({}, error)
{
}
template<typename T>
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator*() noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator*() noexcept {
if (this->error()) { this->second = SUCCESS; return this->error(); }
return *this->first;
}
template<typename T>
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &other) const noexcept {
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &other) const noexcept {
if (this->error()) { return true; }
return this->first == other.first;
}
template<typename T>
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &other) const noexcept {
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &other) const noexcept {
if (this->error()) { return false; }
return this->first != other.first;
}
template<typename T>
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator++() noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator++() noexcept {
if (this->error()) { return *this; }
++(this->first);
return *this;

32
include/simdjson/generic/ondemand/array_iterator.h
View File

@ -16,13 +16,10 @@ class document;
* - * must be called exactly once per element.
* - ++ must be called exactly once in between each * (*, ++, *, ++, * ...)
*/
template<typename T>
class array_iterator {
public:
/** Create a new, invalid array iterator. */
simdjson_really_inline array_iterator() noexcept = default;
simdjson_really_inline array_iterator(const array_iterator<T> &a) noexcept = default;
simdjson_really_inline array_iterator<T> &operator=(const array_iterator<T> &a) noexcept = default;
//
// Iterator interface
@ -41,7 +38,7 @@ public:
*
* @return true if there are no more elements in the JSON array.
*/
simdjson_really_inline bool operator==(const array_iterator<T> &) const noexcept;
simdjson_really_inline bool operator==(const array_iterator &) const noexcept;
/**
* Check if there are more elements in the JSON array.
*
@ -49,25 +46,22 @@ public:
*
* @return true if there are more elements in the JSON array.
*/
simdjson_really_inline bool operator!=(const array_iterator<T> &) const noexcept;
simdjson_really_inline bool operator!=(const array_iterator &) const noexcept;
/**
* Move to the next element.
*
* Part of the std::iterator interface.
*/
simdjson_really_inline array_iterator<T> &operator++() noexcept;
simdjson_really_inline array_iterator &operator++() noexcept;
private:
T *iter{};
value_iterator iter{};
simdjson_really_inline array_iterator(T &iter) noexcept;
simdjson_really_inline array_iterator(const value_iterator &iter) noexcept;
static simdjson_really_inline simdjson_result<array_iterator<T>> start(T &iter, const uint8_t *json) noexcept;
friend T;
friend class array;
friend class value;
friend struct simdjson_result<array_iterator<T>>;
friend struct simdjson_result<array_iterator>;
};
} // namespace ondemand
@ -76,14 +70,14 @@ private:
namespace simdjson {
template<typename T>
struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> {
template<>
struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> {
public:
simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T> &&value) noexcept; ///< @private
simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::array_iterator &&value) noexcept; ///< @private
simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
simdjson_really_inline simdjson_result() noexcept = default;
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &&a) noexcept = default;
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &&a) noexcept = default;
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
//
@ -91,9 +85,9 @@ public:
//
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION.
simdjson_really_inline bool operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &) const noexcept;
simdjson_really_inline bool operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &) const noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &operator++() noexcept;
simdjson_really_inline bool operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &) const noexcept;
simdjson_really_inline bool operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &) const noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &operator++() noexcept;
};
} // namespace simdjson

82
include/simdjson/generic/ondemand/document-inl.h
View File

@ -2,69 +2,58 @@ namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
simdjson_really_inline document::document(ondemand::json_iterator &&_iter, const uint8_t *_json) noexcept
: iter{std::forward<json_iterator>(_iter)},
json{_json}
simdjson_really_inline document::document(ondemand::json_iterator &&_iter) noexcept
: iter{std::forward<json_iterator>(_iter)}
{
logger::log_start_value(iter, "document");
SIMDJSON_ASSUME(json);
}
simdjson_really_inline void document::assert_at_start() const noexcept {
SIMDJSON_ASSUME(json);
SIMDJSON_ASSUME(iter.at_start());
}
simdjson_really_inline document document::start(json_iterator &&iter) noexcept {
SIMDJSON_ASSUME(iter.container_depth == DOCUMENT_DEPTH);
auto json = iter.advance();
return document(std::forward<json_iterator>(iter), json);
return document(std::forward<json_iterator>(iter));
}
simdjson_really_inline value document::as_value() noexcept {
assert_at_start();
SIMDJSON_ASSUME(iter.container_depth == DOCUMENT_DEPTH);
return { json_iterator_ref(&iter, DOCUMENT_DEPTH + 1), json };
return as_value_iterator();
}
template<typename T>
simdjson_result<T> document::consume_if_success(simdjson_result<T> &&result) noexcept {
if (!result.error()) { json = nullptr; }
return std::forward<simdjson_result<T>>(result);
simdjson_really_inline value_iterator document::as_value_iterator() noexcept {
assert_at_start();
return value_iterator(&iter, 1);
}
simdjson_really_inline simdjson_result<array> document::get_array() & noexcept {
assert_at_start();
return consume_if_success( as_value().get_array() );
return as_value().get_array();
}
simdjson_really_inline simdjson_result<object> document::get_object() & noexcept {
assert_at_start();
return consume_if_success( as_value().get_object() );
return as_value().get_object();
}
simdjson_really_inline simdjson_result<uint64_t> document::get_uint64() noexcept {
assert_at_start();
return consume_if_success( iter.parse_root_uint64(json) );
return as_value_iterator().require_root_uint64();
}
simdjson_really_inline simdjson_result<int64_t> document::get_int64() noexcept {
assert_at_start();
return consume_if_success( iter.parse_root_int64(json) );
return as_value_iterator().require_root_int64();
}
simdjson_really_inline simdjson_result<double> document::get_double() noexcept {
assert_at_start();
return consume_if_success( iter.parse_root_double(json) );
return as_value_iterator().require_root_double();
}
simdjson_really_inline simdjson_result<std::string_view> document::get_string() & noexcept {
return consume_if_success( as_value().get_string() );
return as_value().get_string();
}
simdjson_really_inline simdjson_result<raw_json_string> document::get_raw_json_string() & noexcept {
return consume_if_success( as_value().get_raw_json_string() );
return as_value().get_raw_json_string();
}
simdjson_really_inline simdjson_result<bool> document::get_bool() noexcept {
assert_at_start();
return consume_if_success( iter.parse_root_bool(json) );
return as_value_iterator().require_root_bool();
}
simdjson_really_inline bool document::is_null() noexcept {
assert_at_start();
if (iter.root_is_null(json)) { json = nullptr; return true; }
return false;
return as_value_iterator().is_root_null();
}
template<> simdjson_really_inline simdjson_result<array> document::get() & noexcept { return get_array(); }
@ -99,10 +88,10 @@ simdjson_really_inline document::operator raw_json_string() & noexcept(false) {
simdjson_really_inline document::operator bool() noexcept(false) { return get_bool(); }
#endif
simdjson_really_inline simdjson_result<array_iterator<document>> document::begin() & noexcept {
return consume_if_success( array_iterator<document>::start(*this, json) );
simdjson_really_inline simdjson_result<array_iterator> document::begin() & noexcept {
return get_array().begin();
}
simdjson_really_inline simdjson_result<array_iterator<document>> document::end() & noexcept {
simdjson_really_inline simdjson_result<array_iterator> document::end() & noexcept {
return {};
}
simdjson_really_inline simdjson_result<value> document::operator[](std::string_view key) & noexcept {
@ -112,33 +101,6 @@ simdjson_really_inline simdjson_result<value> document::operator[](const char *k
return get_object()[key];
}
//
// For array_iterator
//
simdjson_really_inline json_iterator &document::get_iterator() noexcept {
return iter;
}
simdjson_really_inline json_iterator_ref document::borrow_iterator_child() noexcept {
SIMDJSON_ASSUME(iter.container_depth == DOCUMENT_DEPTH + 1);
return json_iterator_ref(&iter, DOCUMENT_DEPTH);
}
simdjson_really_inline bool document::is_iterator_alive() const noexcept {
return iter.container_depth != DOCUMENT_DEPTH;
}
simdjson_really_inline void document::start_iterator() noexcept {
SIMDJSON_ASSUME(iter.container_depth == DOCUMENT_DEPTH + 1);
}
simdjson_really_inline void document::finish_iterator() noexcept {
SIMDJSON_ASSUME(iter.container_depth == DOCUMENT_DEPTH);
}
simdjson_really_inline void document::abandon_iterator() noexcept {
iter.container_depth = DOCUMENT_DEPTH;
}
simdjson_warn_unused simdjson_really_inline error_code document::finish_iterator_child() noexcept {
SIMDJSON_ASSUME(iter.container_depth > DOCUMENT_DEPTH);
return iter.finish_child(DOCUMENT_DEPTH + 1);
}
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
@ -162,11 +124,11 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::docume
{
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::document>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::begin() & noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::begin() & noexcept {
if (error()) { return error(); }
return first.begin();
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::document>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::end() & noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::end() & noexcept {
return {};
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator[](std::string_view key) & noexcept {

36
include/simdjson/generic/ondemand/document.h
View File

@ -9,7 +9,7 @@ class array;
class object;
class value;
class raw_json_string;
template<typename T> class array_iterator;
class array_iterator;
/**
* A JSON document iteration.
@ -198,13 +198,13 @@ public:
*
* Part of the std::iterable interface.
*/
simdjson_really_inline simdjson_result<array_iterator<document>> begin() & noexcept;
simdjson_really_inline simdjson_result<array_iterator> begin() & noexcept;
/**
* Sentinel representing the end of the array.
*
* Part of the std::iterable interface.
*/
simdjson_really_inline simdjson_result<array_iterator<document>> end() & noexcept;
simdjson_really_inline simdjson_result<array_iterator> end() & noexcept;
/**
* Look up a field by name on an object.
@ -228,41 +228,23 @@ public:
simdjson_really_inline simdjson_result<value> operator[](const char *key) & noexcept;
protected:
simdjson_really_inline document(ondemand::json_iterator &&iter, const uint8_t *json) noexcept;
simdjson_really_inline document(ondemand::json_iterator &&iter) noexcept;
simdjson_really_inline const uint8_t *text(uint32_t idx) const noexcept;
simdjson_really_inline value as_value() noexcept;
simdjson_really_inline value_iterator as_value_iterator() noexcept;
static simdjson_really_inline document start(ondemand::json_iterator &&iter) noexcept;
/**
* Set json to null if the result is successful.
*
* Convenience function for value-getters.
*/
template<typename T>
simdjson_result<T> consume_if_success(simdjson_result<T> &&result) noexcept;
simdjson_really_inline void assert_at_start() const noexcept;
//
// For array_iterator
//
simdjson_really_inline json_iterator &get_iterator() noexcept;
simdjson_really_inline json_iterator_ref borrow_iterator_child() noexcept;
simdjson_really_inline bool is_iterator_alive() const noexcept;
simdjson_really_inline void start_iterator() noexcept;
simdjson_really_inline void finish_iterator() noexcept;
simdjson_really_inline void abandon_iterator() noexcept;
simdjson_warn_unused simdjson_really_inline error_code finish_iterator_child() noexcept;
//
// Fields
//
json_iterator iter{}; ///< Current position in the document
const uint8_t *json{}; ///< JSON for the value in the document (nullptr if value has been consumed)
static constexpr uint32_t DOCUMENT_DEPTH = 0; ///< document depth is always 0
static constexpr depth_t DOCUMENT_DEPTH = 0; ///< document depth is always 0
friend struct simdjson_result<document>;
friend class array_iterator<document>;
friend class array_iterator;
friend class value;
friend class ondemand::parser;
friend class object;
@ -314,8 +296,8 @@ public:
simdjson_really_inline operator bool() noexcept(false);
#endif
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::document>> begin() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::document>> end() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
};

14
include/simdjson/generic/ondemand/field-inl.h
View File

@ -10,20 +10,20 @@ simdjson_really_inline field::field(raw_json_string key, ondemand::value &&value
{
}
simdjson_really_inline simdjson_result<field> field::start(json_iterator_ref &parent_iter) noexcept {
simdjson_really_inline simdjson_result<field> field::start(value_iterator &parent_iter) noexcept {
raw_json_string key;
SIMDJSON_TRY( parent_iter->field_key().get(key) );
SIMDJSON_TRY( parent_iter->field_value() );
return field::start(parent_iter.borrow(), key);
SIMDJSON_TRY( parent_iter.field_key().get(key) );
SIMDJSON_TRY( parent_iter.field_value() );
return field::start(parent_iter, key);
}
simdjson_really_inline simdjson_result<field> field::start(json_iterator_ref &&iter, raw_json_string key) noexcept {
return field(key, value::start(std::forward<json_iterator_ref>(iter)));
simdjson_really_inline simdjson_result<field> field::start(const value_iterator &parent_iter, raw_json_string key) noexcept {
return field(key, parent_iter.child());
}
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key() noexcept {
SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us.
simdjson_result<std::string_view> answer = first.unescape(second.get_iterator());
simdjson_result<std::string_view> answer = first.unescape(second.iter.string_buf_loc());
first.consume();
return answer;
}

4
include/simdjson/generic/ondemand/field.h
View File

@ -50,8 +50,8 @@ public:
protected:
simdjson_really_inline field(raw_json_string key, ondemand::value &&value) noexcept;
static simdjson_really_inline simdjson_result<field> start(json_iterator_ref &iter) noexcept;
static simdjson_really_inline simdjson_result<field> start(json_iterator_ref &&iter, raw_json_string key) noexcept;
static simdjson_really_inline simdjson_result<field> start(value_iterator &parent_iter) noexcept;
static simdjson_really_inline simdjson_result<field> start(const value_iterator &parent_iter, raw_json_string key) noexcept;
friend struct simdjson_result<field>;
friend class object_iterator;
};

420
include/simdjson/generic/ondemand/json_iterator-inl.h
View File

@ -3,273 +3,51 @@ namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
simdjson_really_inline json_iterator::json_iterator(json_iterator &&other) noexcept
: token_iterator(std::forward<token_iterator>(other)),
: token(std::forward<token_iterator>(other.token)),
parser{other.parser},
current_string_buf_loc{other.current_string_buf_loc},
container_depth{other.container_depth}
_string_buf_loc{other._string_buf_loc},
_depth{other._depth}
{
other.parser = nullptr;
}
simdjson_really_inline json_iterator &json_iterator::operator=(json_iterator &&other) noexcept {
buf = other.buf;
index = other.index;
token = other.token;
parser = other.parser;
current_string_buf_loc = other.current_string_buf_loc;
container_depth = other.container_depth;
_string_buf_loc = other._string_buf_loc;
_depth = other._depth;
other.parser = nullptr;
return *this;
}
simdjson_really_inline json_iterator::json_iterator(ondemand::parser *_parser) noexcept
: token_iterator(_parser->dom_parser.buf, _parser->dom_parser.structural_indexes.get()),
: token(_parser->dom_parser.buf, _parser->dom_parser.structural_indexes.get()),
parser{_parser},
current_string_buf_loc{parser->string_buf.get()},
container_depth{0}
_string_buf_loc{parser->string_buf.get()},
_depth{1}
{
// Release the string buf so it can be reused by the next document
logger::log_headers();
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> json_iterator::start_object(const uint8_t *json) noexcept {
if (*json != '{') { logger::log_error(*this, "Not an object"); return INCORRECT_TYPE; }
return started_object();
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> json_iterator::start_object() noexcept {
return start_object(advance());
}
simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept {
SIMDJSON_ASSUME(depth() == parent_depth+1);
simdjson_warn_unused simdjson_really_inline bool json_iterator::started_object() noexcept {
if (*peek() == '}') {
logger::log_value(*this, "empty object");
advance();
return false;
}
container_depth++;
logger::log_start_value(*this, "object");
return true;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> json_iterator::has_next_field() noexcept {
switch (*advance()) {
case '}':
logger::log_end_value(*this, "object");
container_depth--;
return false;
case ',':
return true;
default:
return report_error(TAPE_ERROR, "Missing comma between object fields");
}
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> json_iterator::find_field_raw(const char *key) noexcept {
bool has_next;
do {
raw_json_string actual_key;
SIMDJSON_TRY( consume_raw_json_string().get(actual_key) );
if (*advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); }
if (actual_key == key) {
logger::log_event(*this, "match", key);
return true;
}
logger::log_event(*this, "non-match", key);
SIMDJSON_TRY( skip() ); // Skip the value so we can look at the next key
SIMDJSON_TRY( has_next_field().get(has_next) );
} while (has_next);
logger::log_event(*this, "no matches", key);
return false;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> json_iterator::field_key() noexcept {
const uint8_t *key = advance();
if (*(key++) != '"') { return report_error(TAPE_ERROR, "Object key is not a string"); }
return raw_json_string(key);
}
simdjson_warn_unused simdjson_really_inline error_code json_iterator::field_value() noexcept {
if (*advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); }
return SUCCESS;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> json_iterator::start_array(const uint8_t *json) noexcept {
if (*json != '[') { logger::log_error(*this, "Not an array"); return INCORRECT_TYPE; }
return started_array();
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> json_iterator::start_array() noexcept {
return start_array(advance());
}
simdjson_warn_unused simdjson_really_inline bool json_iterator::started_array() noexcept {
if (*peek() == ']') {
logger::log_value(*this, "empty array");
advance();
return false;
}
logger::log_start_value(*this, "array");
container_depth++;
return true;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> json_iterator::has_next_element() noexcept {
switch (*advance()) {
case ']':
logger::log_end_value(*this, "array");
container_depth--;
return false;
case ',':
return true;
default:
return report_error(TAPE_ERROR, "Missing comma between array elements");
}
}
simdjson_warn_unused simdjson_result<std::string_view> json_iterator::parse_string(const uint8_t *json) noexcept {
return parse_raw_json_string(json).unescape(current_string_buf_loc);
}
simdjson_warn_unused simdjson_result<std::string_view> json_iterator::consume_string() noexcept {
return parse_string(advance());
}
simdjson_warn_unused simdjson_result<raw_json_string> json_iterator::parse_raw_json_string(const uint8_t *json) noexcept {
logger::log_value(*this, "string", "");
if (*json != '"') { logger::log_error(*this, "Not a string"); return INCORRECT_TYPE; }
return raw_json_string(json+1);
}
simdjson_warn_unused simdjson_result<raw_json_string> json_iterator::consume_raw_json_string() noexcept {
return parse_raw_json_string(advance());
}
simdjson_warn_unused simdjson_result<uint64_t> json_iterator::parse_uint64(const uint8_t *json) noexcept {
logger::log_value(*this, "uint64", "");
return numberparsing::parse_unsigned(json);
}
simdjson_warn_unused simdjson_result<uint64_t> json_iterator::consume_uint64() noexcept {
return parse_uint64(advance());
}
simdjson_warn_unused simdjson_result<int64_t> json_iterator::parse_int64(const uint8_t *json) noexcept {
logger::log_value(*this, "int64", "");
return numberparsing::parse_integer(json);
}
simdjson_warn_unused simdjson_result<int64_t> json_iterator::consume_int64() noexcept {
return parse_int64(advance());
}
simdjson_warn_unused simdjson_result<double> json_iterator::parse_double(const uint8_t *json) noexcept {
logger::log_value(*this, "double", "");
return numberparsing::parse_double(json);
}
simdjson_warn_unused simdjson_result<double> json_iterator::consume_double() noexcept {
return parse_double(advance());
}
simdjson_warn_unused simdjson_result<bool> json_iterator::parse_bool(const uint8_t *json) noexcept {
logger::log_value(*this, "bool", "");
auto not_true = atomparsing::str4ncmp(json, "true");
auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
if (error) { logger::log_error(*this, "Not a boolean"); return INCORRECT_TYPE; }
return simdjson_result<bool>(!not_true);
}
simdjson_warn_unused simdjson_result<bool> json_iterator::consume_bool() noexcept {
return parse_bool(advance());
}
simdjson_really_inline bool json_iterator::is_null(const uint8_t *json) noexcept {
if (!atomparsing::str4ncmp(json, "null")) {
logger::log_value(*this, "null", "");
return true;
}
return false;
}
simdjson_really_inline bool json_iterator::is_null() noexcept {
if (is_null(peek())) {
advance();
return true;
}
return false;
}
template<int N>
simdjson_warn_unused simdjson_really_inline bool json_iterator::copy_to_buffer(const uint8_t *json, uint8_t (&tmpbuf)[N]) noexcept {
// Truncate whitespace to fit the buffer.
auto len = peek_length(-1);
if (len > N-1) {
if (jsoncharutils::is_not_structural_or_whitespace(json[N])) { return false; }
len = N-1;
}
// Copy to the buffer.
std::memcpy(tmpbuf, json, len);
tmpbuf[len] = ' ';
return true;
}
constexpr const uint32_t MAX_INT_LENGTH = 1024;
simdjson_warn_unused simdjson_result<uint64_t> json_iterator::parse_root_uint64(const uint8_t *json) noexcept {
uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer
if (!copy_to_buffer(json, tmpbuf)) { logger::log_error(*this, "Root number more than 20 characters"); return NUMBER_ERROR; }
logger::log_value(*this, "uint64", "");
auto result = numberparsing::parse_unsigned(tmpbuf);
if (result.error()) { logger::log_error(*this, "Error parsing unsigned integer"); return result.error(); }
return result;
}
simdjson_warn_unused simdjson_result<uint64_t> json_iterator::consume_root_uint64() noexcept {
return parse_root_uint64(advance());
}
simdjson_warn_unused simdjson_result<int64_t> json_iterator::parse_root_int64(const uint8_t *json) noexcept {
uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer
if (!copy_to_buffer(json, tmpbuf)) { logger::log_error(*this, "Root number more than 20 characters"); return NUMBER_ERROR; }
logger::log_value(*this, "int64", "");
auto result = numberparsing::parse_integer(tmpbuf);
if (result.error()) { report_error(result.error(), "Error parsing integer"); }
return result;
}
simdjson_warn_unused simdjson_result<int64_t> json_iterator::consume_root_int64() noexcept {
return parse_root_int64(advance());
}
simdjson_warn_unused simdjson_result<double> json_iterator::parse_root_double(const uint8_t *json) noexcept {
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest number: -0.<fraction>e-308.
uint8_t tmpbuf[1074+8+1];
if (!copy_to_buffer(json, tmpbuf)) { logger::log_error(*this, "Root number more than 1082 characters"); return NUMBER_ERROR; }
logger::log_value(*this, "double", "");
auto result = numberparsing::parse_double(tmpbuf);
if (result.error()) { report_error(result.error(), "Error parsing double"); }
return result;
}
simdjson_warn_unused simdjson_result<double> json_iterator::consume_root_double() noexcept {
return parse_root_double(advance());
}
simdjson_warn_unused simdjson_result<bool> json_iterator::parse_root_bool(const uint8_t *json) noexcept {
uint8_t tmpbuf[5+1];
if (!copy_to_buffer(json, tmpbuf)) { logger::log_error(*this, "Not a boolean"); return INCORRECT_TYPE; }
return parse_bool(tmpbuf);
}
simdjson_warn_unused simdjson_result<bool> json_iterator::consume_root_bool() noexcept {
return parse_root_bool(advance());
}
simdjson_really_inline bool json_iterator::root_is_null(const uint8_t *json) noexcept {
uint8_t tmpbuf[4+1];
if (!copy_to_buffer(json, tmpbuf)) { return false; }
return is_null(tmpbuf);
}
simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip() noexcept {
switch (*advance()) {
// PERF TODO does it skip the depth check when we don't decrement depth?
case '[': case '{':
container_depth++;
logger::log_start_value(*this, "skip");
return skip_container();
return finish_child(parent_depth);
default:
logger::log_value(*this, "skip", "");
logger::log_value(*this, "skip");
ascend_to(parent_depth);
return SUCCESS;
}
}
simdjson_warn_unused simdjson_really_inline error_code json_iterator::finish_child(uint32_t depth) noexcept {
uint32_t relative_depth = container_depth - depth;
if (relative_depth == 0) { return SUCCESS; }
// The loop breaks only when depth-- happens.
simdjson_warn_unused simdjson_really_inline error_code json_iterator::finish_child(depth_t parent_depth) noexcept {
if (depth() <= parent_depth) { return SUCCESS; }
// The loop breaks only when depth()-- happens.
auto end = &parser->dom_parser.structural_indexes[parser->dom_parser.n_structural_indexes];
while (index <= end) {
while (token.index <= end) {
uint8_t ch = *advance();
switch (ch) {
// TODO consider whether matching braces is a requirement: if non-matching braces indicates
@ -278,13 +56,13 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::finish_chi
// looking at the right values."
case ']': case '}':
logger::log_end_value(*this, "skip");
relative_depth--;
if (relative_depth == 0) { container_depth = depth; return SUCCESS; }
_depth--;
if (depth() <= parent_depth) { return SUCCESS; }
break;
// PERF TODO does it skip the depth check when we don't decrement depth?
// PERF TODO does it skip the depth() check when we don't decrement depth()?
case '[': case '{':
logger::log_start_value(*this, "skip");
relative_depth++;
_depth++;
break;
default:
logger::log_value(*this, "skip", "");
@ -295,110 +73,93 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::finish_chi
return report_error(TAPE_ERROR, "not enough close braces");
}
simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_container() noexcept {
return finish_child(container_depth-1);
}
simdjson_really_inline bool json_iterator::at_start() const noexcept {
return index == parser->dom_parser.structural_indexes.get();
return token.index == parser->dom_parser.structural_indexes.get();
}
simdjson_really_inline bool json_iterator::at_eof() const noexcept {
return index == &parser->dom_parser.structural_indexes[parser->dom_parser.n_structural_indexes];
return token.index == &parser->dom_parser.structural_indexes[parser->dom_parser.n_structural_indexes];
}
simdjson_really_inline bool json_iterator::is_alive() const noexcept {
return parser;
}
simdjson_really_inline error_code json_iterator::report_error(error_code error, const char *message) noexcept {
SIMDJSON_ASSUME(error != SUCCESS && error != UNINITIALIZED && error != INCORRECT_TYPE && error != NO_SUCH_FIELD);
simdjson_really_inline void json_iterator::abandon() noexcept {
parser = nullptr;
_depth = 0;
}
simdjson_really_inline const uint8_t *json_iterator::advance() noexcept {
return token.advance();
}
simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept {
return token.peek(delta);
}
simdjson_really_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept {
return token.peek_length(delta);
}
simdjson_really_inline void json_iterator::ascend_to(depth_t parent_depth) noexcept {
SIMDJSON_ASSUME(depth() == parent_depth + 1);
_depth = parent_depth;
}
simdjson_really_inline void json_iterator::descend_to(depth_t child_depth) noexcept {
SIMDJSON_ASSUME(depth() == child_depth - 1);
_depth = child_depth;
}
simdjson_really_inline depth_t json_iterator::depth() const noexcept {
return _depth;
}
simdjson_really_inline uint8_t *&json_iterator::string_buf_loc() noexcept {
return _string_buf_loc;
}
simdjson_really_inline error_code json_iterator::report_error(error_code _error, const char *message) noexcept {
SIMDJSON_ASSUME(_error != SUCCESS && _error != UNINITIALIZED && _error != INCORRECT_TYPE && _error != NO_SUCH_FIELD);
logger::log_error(*this, message);
_error = error;
error = _error;
return error;
}
simdjson_really_inline error_code json_iterator::error() const noexcept {
simdjson_really_inline error_code json_iterator::optional_error(error_code _error, const char *message) noexcept {
SIMDJSON_ASSUME(_error == INCORRECT_TYPE || _error == NO_SUCH_FIELD);
logger::log_error(*this, message);
return _error;
}
//
// json_iterator_ref
//
simdjson_really_inline json_iterator_ref::json_iterator_ref(json_iterator_ref &&other) noexcept
: iter{other.iter},
depth{other.depth}
{
other.iter = nullptr;
}
simdjson_really_inline json_iterator_ref &json_iterator_ref::operator=(json_iterator_ref &&other) noexcept {
assert_is_not_active();
iter = other.iter;
depth = other.depth;
other.iter = nullptr;
return *this;
template<int N>
simdjson_warn_unused simdjson_really_inline bool json_iterator::copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept {
// Truncate whitespace to fit the buffer.
if (max_len > N-1) {
if (jsoncharutils::is_not_structural_or_whitespace(json[N])) { return false; }
max_len = N-1;
}
// Copy to the buffer.
std::memcpy(tmpbuf, json, max_len);
tmpbuf[max_len] = ' ';
return true;
}
simdjson_really_inline json_iterator_ref::json_iterator_ref(
json_iterator *_iter,
uint32_t _depth
) noexcept : iter{_iter}, depth{_depth}
{
assert_is_active();
template<int N>
simdjson_warn_unused simdjson_really_inline bool json_iterator::peek_to_buffer(uint8_t (&tmpbuf)[N]) noexcept {
auto max_len = token.peek_length();
auto json = token.peek();
return copy_to_buffer(json, max_len, tmpbuf);
}
simdjson_really_inline json_iterator_ref json_iterator_ref::borrow() noexcept {
assert_is_active();
return json_iterator_ref(iter, depth + 1);
template<int N>
simdjson_warn_unused simdjson_really_inline bool json_iterator::advance_to_buffer(uint8_t (&tmpbuf)[N]) noexcept {
auto max_len = peek_length();
auto json = advance();
return copy_to_buffer(json, max_len, tmpbuf);
}
simdjson_really_inline void json_iterator_ref::started_container() noexcept {
assert_is_active();
SIMDJSON_ASSUME(iter->container_depth == depth);
}
simdjson_really_inline void json_iterator_ref::finished_container() noexcept {
assert_is_active();
SIMDJSON_ASSUME(iter->container_depth == depth - 1);
iter = nullptr;
}
simdjson_really_inline void json_iterator_ref::abandon() noexcept {
assert_is_active();
iter = nullptr;
}
simdjson_really_inline json_iterator *json_iterator_ref::operator->() noexcept {
assert_is_active();
return iter;
}
simdjson_really_inline json_iterator &json_iterator_ref::operator*() noexcept {
assert_is_active();
return *iter;
}
simdjson_really_inline const json_iterator &json_iterator_ref::operator*() const noexcept {
assert_is_active();
return *iter;
}
simdjson_warn_unused simdjson_really_inline error_code json_iterator_ref::finish_child() noexcept {
assert_is_active();
return iter->finish_child(depth);
}
simdjson_really_inline bool json_iterator_ref::is_alive() const noexcept {
return iter != nullptr;
}
simdjson_really_inline bool json_iterator_ref::is_active() const noexcept {
return is_alive() && depth == iter->container_depth;
}
simdjson_really_inline void json_iterator_ref::assert_is_active() const noexcept {
// We don't call const functions because VC++ is worried they might have side effects in __assume
// TODO fix depth check
SIMDJSON_ASSUME(iter != nullptr); // && depth == iter->container_depth);
}
simdjson_really_inline void json_iterator_ref::assert_is_not_active() const noexcept {
// We don't call const functions because VC++ is worried they might have side effects in __assume
SIMDJSON_ASSUME(!(iter != nullptr)); // && depth == iter->container_depth));
}
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
@ -411,9 +172,4 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_i
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator>::simdjson_result(error_code error) noexcept
: implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator>(error) {}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref>::simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref &&value) noexcept
: implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref>(value)) {}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref>::simdjson_result(error_code error) noexcept
: implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref>(error) {}
} // namespace simdjson

314
include/simdjson/generic/ondemand/json_iterator.h
View File

@ -8,14 +8,43 @@ class array;
class value;
class raw_json_string;
class parser;
class json_iterator_ref;
/**
* Iterates through JSON, with structure-sensitive algorithms.
* Iterates through JSON tokens, keeping track of depth and string buffer.
*
* @private This is not intended for external use.
*/
class json_iterator : public token_iterator {
class json_iterator {
protected:
token_iterator token{};
ondemand::parser *parser{};
/**
* Next free location in the string buffer.
*
* Used by raw_json_string::unescape() to have a place to unescape strings to.
*/
uint8_t *_string_buf_loc{};
/**
* JSON error, if there is one.
*
* INCORRECT_TYPE and NO_SUCH_FIELD are *not* stored here, ever.
*
* PERF NOTE: we *hope* this will be elided into control flow, as it is only used (a) in the first
* iteration of the loop, or (b) for the final iteration after a missing comma is found in ++. If
* this is not elided, we should make sure it's at least not using up a register. Failing that,
* we should store it in document so there's only one of them.
*/
error_code error{SUCCESS};
/**
* Depth of the current token in the JSON.
*
* - 0 = finished with document
* - 1 = document root value (could be [ or {, not yet known)
* - 2 = , or } inside root array/object
* - 3 = key or value inside root array/object.
*/
depth_t _depth{};
public:
simdjson_really_inline json_iterator() noexcept = default;
simdjson_really_inline json_iterator(json_iterator &&other) noexcept;
@ -23,141 +52,15 @@ public:
simdjson_really_inline json_iterator(const json_iterator &other) noexcept = delete;
simdjson_really_inline json_iterator &operator=(const json_iterator &other) noexcept = delete;
/**
* Check for an opening { and start an object iteration.
*
* @param json A pointer to the potential {
* @returns Whether the object had any fields (returns false for empty).
* @error INCORRECT_TYPE if there is no opening {
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> start_object(const uint8_t *json) noexcept;
/**
* Check for an opening { and start an object iteration.
*
* @returns Whether the object had any fields (returns false for empty).
* @error INCORRECT_TYPE if there is no opening {
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> start_object() noexcept;
/**
* Start an object iteration after the user has already checked and moved past the {.
*
* Does not move the iterator.
*
* @returns Whether the object had any fields (returns false for empty).
*/
simdjson_warn_unused simdjson_really_inline bool started_object() noexcept;
/**
* Moves to the next field in an object.
*
* Looks for , and }. If } is found, the object is finished and the iterator advances past it.
* Otherwise, it advances to the next value.
*
* @return whether there is another field in the object.
* @error TAPE_ERROR If there is a comma missing between fields.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> has_next_field() noexcept;
/**
* Get the current field's key.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> field_key() noexcept;
/**
* Pass the : in the field and move to its value.
*/
simdjson_warn_unused simdjson_really_inline error_code field_value() noexcept;
/**
* Find the next field with the given key.
*
* Assumes you have called next_field() or otherwise matched the previous value.
*
* Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to
* unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may
* fail to match some keys with escapes (\u, \n, etc.).
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> find_field_raw(const char *key) noexcept;
/**
* Check for an opening [ and start an array iteration.
*
* @param json A pointer to the potential [.
* @returns Whether the array had any elements (returns false for empty).
* @error INCORRECT_TYPE If there is no [.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> start_array(const uint8_t *json) noexcept;
/**
* Check for an opening [ and start an array iteration.
*
* @returns Whether the array had any elements (returns false for empty).
* @error INCORRECT_TYPE If there is no [.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> start_array() noexcept;
/**
* Start an array iteration after the user has already checked and moved past the [.
*
* Does not move the iterator.
*
* @returns Whether the array had any elements (returns false for empty).
*/
simdjson_warn_unused simdjson_really_inline bool started_array() noexcept;
/**
* Moves to the next element in an array.
*
* Looks for , and ]. If ] is found, the array is finished and the iterator advances past it.
* Otherwise, it advances to the next value.
*
* @return Whether there is another element in the array.
* @error TAPE_ERROR If there is a comma missing between elements.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> has_next_element() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> parse_string(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> consume_string() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> parse_raw_json_string(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> consume_raw_json_string() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> parse_uint64(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> consume_uint64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> parse_int64(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> consume_int64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> consume_double() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> parse_bool(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> consume_bool() noexcept;
simdjson_really_inline bool is_null(const uint8_t *json) noexcept;
simdjson_really_inline bool is_null() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> parse_root_uint64(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> consume_root_uint64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> parse_root_int64(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> consume_root_int64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> parse_root_double(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> consume_root_double() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> parse_root_bool(const uint8_t *json) noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> consume_root_bool() noexcept;
simdjson_really_inline bool root_is_null(const uint8_t *json) noexcept;
simdjson_really_inline bool root_is_null() noexcept;
/**
* Skips a JSON value, whether it is a scalar, array or object.
*/
simdjson_warn_unused simdjson_really_inline error_code skip() noexcept;
simdjson_warn_unused simdjson_really_inline error_code skip_child(depth_t parent_depth) noexcept;
/**
* Finishes iteration of a child in an object or array.
*/
simdjson_warn_unused simdjson_really_inline error_code finish_child(uint32_t depth) noexcept;
/**
* Skips to the end of a JSON object or array.
*
* @return true if this was the end of an array, false if it was the end of an object.
*/
simdjson_warn_unused simdjson_really_inline error_code skip_container() noexcept;
simdjson_warn_unused simdjson_really_inline error_code finish_child(depth_t parent_depth) noexcept;
/**
* Tell whether the iterator is still at the start
@ -174,6 +77,68 @@ public:
*/
simdjson_really_inline bool is_alive() const noexcept;
/**
* Abandon this iterator, setting depth to 0 (as if the document is finished).
*/
simdjson_really_inline void abandon() noexcept;
/**
* Advance the current token.
*/
simdjson_really_inline const uint8_t *advance() noexcept;
/**
* Whether we are at the start of an object.
*/
/**
* Get the JSON text for a given token (relative).
*
* This is not null-terminated; it is a view into the JSON.
*
* @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token.
*
* TODO consider a string_view, assuming the length will get stripped out by the optimizer when
* it isn't used ...
*/
simdjson_really_inline const uint8_t *peek(int32_t delta=0) const noexcept;
/**
* Get the maximum length of the JSON text for a given token.
*
* The length will include any whitespace at the end of the token.
*
* @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token.
*/
simdjson_really_inline uint32_t peek_length(int32_t delta=0) const noexcept;
/**
* Ascend one level.
*
* Validates that the depth - 1 == parent_depth.
*
* @param parent_depth the expected parent depth.
*/
simdjson_really_inline void ascend_to(depth_t parent_depth) noexcept;
/**
* Descend one level.
*
* Validates that the new depth == child_depth.
*
* @param child_depth the expected child depth.
*/
simdjson_really_inline void descend_to(depth_t parent_depth) noexcept;
/**
* Get current depth.
*/
simdjson_really_inline depth_t depth() const noexcept;
/**
* Get current (writeable) location in the string buffer.
*/
simdjson_really_inline uint8_t *&string_buf_loc() noexcept;
/**
* Report an error, preventing further iteration.
*
@ -183,34 +148,18 @@ public:
simdjson_really_inline error_code report_error(error_code error, const char *message) noexcept;
/**
* Get the error (if any).
* Log error, but don't stop iteration.
* @param error The error to report. Must be INCORRECT_TYPE, or NO_SUCH_FIELD.
* @param message An error message to report with the error.
*/
simdjson_really_inline error_code error() const noexcept;
simdjson_really_inline error_code optional_error(error_code error, const char *message) noexcept;
template<int N> simdjson_warn_unused simdjson_really_inline bool copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept;
template<int N> simdjson_warn_unused simdjson_really_inline bool peek_to_buffer(uint8_t (&tmpbuf)[N]) noexcept;
template<int N> simdjson_warn_unused simdjson_really_inline bool advance_to_buffer(uint8_t (&tmpbuf)[N]) noexcept;
protected:
ondemand::parser *parser{};
/**
* Next free location in the string buffer.
*
* Used by raw_json_string::unescape() to have a place to unescape strings to.
*/
uint8_t *current_string_buf_loc{};
/**
* JSON error, if there is one.
*
* INCORRECT_TYPE and NO_SUCH_FIELD are *not* stored here, ever.
*
* PERF NOTE: we *hope* this will be elided into control flow, as it is only used (a) in the first
* iteration of the loop, or (b) for the final iteration after a missing comma is found in ++. If
* this is not elided, we should make sure it's at least not using up a register. Failing that,
* we should store it in document so there's only one of them.
*/
error_code _error{};
uint32_t container_depth{};
simdjson_really_inline json_iterator(ondemand::parser *parser) noexcept;
template<int N>
simdjson_warn_unused simdjson_really_inline bool copy_to_buffer(const uint8_t *json, uint8_t (&buf)[N]) noexcept;
friend class document;
friend class object;
@ -218,52 +167,10 @@ protected:
friend class value;
friend class raw_json_string;
friend class parser;
friend class json_iterator_ref;
friend class value_iterator;
friend simdjson_really_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
}; // json_iterator
class json_iterator_ref {
public:
simdjson_really_inline json_iterator_ref() noexcept = default;
simdjson_really_inline json_iterator_ref(json_iterator_ref &&other) noexcept;
simdjson_really_inline json_iterator_ref &operator=(json_iterator_ref &&other) noexcept;
simdjson_really_inline json_iterator_ref(const json_iterator_ref &other) noexcept = delete;
simdjson_really_inline json_iterator_ref &operator=(const json_iterator_ref &other) noexcept = delete;
/** Borrow this iterator, incrementing depth */
simdjson_really_inline json_iterator_ref borrow() noexcept;
/** Identify this iterator as a container, incrementing container_depth */
simdjson_really_inline void started_container() noexcept;
/** Denote the container as having finished iterating, decrementing container_depth */
simdjson_really_inline void finished_container() noexcept;
/** Release this iterator without modifying depth */
simdjson_really_inline void abandon() noexcept;
simdjson_really_inline json_iterator *operator->() noexcept;
simdjson_really_inline json_iterator &operator*() noexcept;
simdjson_really_inline const json_iterator &operator*() const noexcept;
simdjson_really_inline bool is_alive() const noexcept;
simdjson_really_inline bool is_active() const noexcept;
simdjson_really_inline void assert_is_active() const noexcept;
simdjson_really_inline void assert_is_not_active() const noexcept;
/**
* Finishes iteration of a child in an object or array.
*/
simdjson_warn_unused simdjson_really_inline error_code finish_child() noexcept;
private:
json_iterator *iter{};
uint32_t depth{};
simdjson_really_inline json_iterator_ref(json_iterator *iter, uint32_t depth) noexcept;
friend class json_iterator;
friend class document;
}; // class json_iterator_ref
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
@ -281,15 +188,4 @@ public:
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
};
template<>
struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref> {
public:
simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref &&value) noexcept; ///< @private
simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
simdjson_really_inline simdjson_result() noexcept = default;
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator_ref> &&a) noexcept = default;
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
};
} // namespace simdjson

18
include/simdjson/generic/ondemand/logger-inl.h
View File

@ -36,6 +36,22 @@ simdjson_really_inline void log_error(const json_iterator &iter, const char *err
log_line(iter, "ERROR: ", error, detail, delta, depth_delta);
}
simdjson_really_inline void log_event(const value_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
log_event(iter.json_iter(), type, detail, delta, depth_delta);
}
simdjson_really_inline void log_value(const value_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
log_value(iter.json_iter(), type, detail, delta, depth_delta);
}
simdjson_really_inline void log_start_value(const value_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
log_start_value(iter.json_iter(), type, delta, depth_delta);
}
simdjson_really_inline void log_end_value(const value_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
log_end_value(iter.json_iter(), type, delta, depth_delta);
}
simdjson_really_inline void log_error(const value_iterator &iter, const char *error, const char *detail, int delta, int depth_delta) noexcept {
log_error(iter.json_iter(), error, detail, delta, depth_delta);
}
simdjson_really_inline void log_headers() noexcept {
log_depth = 0;
if (LOG_ENABLED) {
@ -70,7 +86,7 @@ simdjson_really_inline void log_line(const json_iterator &iter, const char *titl
}
printf(" ");
}
printf("| %5u ", iter.peek_index(delta+1));
printf("| %5u ", iter.token.peek_index(delta+1));
printf("| %.*s ", int(detail.size()), detail.data());
printf("|\n");
fflush(stdout);

7
include/simdjson/generic/ondemand/logger.h
View File

@ -3,6 +3,7 @@ namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
class json_iterator;
class value_iterator;
namespace logger {
@ -20,6 +21,12 @@ static simdjson_really_inline void log_start_value(const json_iterator &iter, co
static simdjson_really_inline void log_end_value(const json_iterator &iter, const char *type, int delta=-1, int depth_delta=0) noexcept;
static simdjson_really_inline void log_error(const json_iterator &iter, const char *error, const char *detail="", int delta=-1, int depth_delta=0) noexcept;
static simdjson_really_inline void log_event(const value_iterator &iter, const char *type, std::string_view detail="", int delta=-1, int depth_delta=0) noexcept;
static simdjson_really_inline void log_value(const value_iterator &iter, const char *type, std::string_view detail="", int delta=-1, int depth_delta=0) noexcept;
static simdjson_really_inline void log_start_value(const value_iterator &iter, const char *type, int delta=-1, int depth_delta=0) noexcept;
static simdjson_really_inline void log_end_value(const value_iterator &iter, const char *type, int delta=-1, int depth_delta=0) noexcept;
static simdjson_really_inline void log_error(const value_iterator &iter, const char *error, const char *detail="", int delta=-1, int depth_delta=0) noexcept;
} // namespace logger
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION

102
include/simdjson/generic/ondemand/object-inl.h
View File

@ -5,31 +5,31 @@ namespace ondemand {
//
// ### Live States
//
// While iterating or looking up values, depth >= iter->depth. at_start may vary. Error is
// While iterating or looking up values, depth >= iter.depth. at_start may vary. Error is
// always SUCCESS:
//
// - Start: This is the state when the object is first found and the iterator is just past the {.
// In this state, at_start == true.
// - Next: After we hand a scalar value to the user, or an array/object which they then fully
// iterate over, the iterator is at the , or } before the next value. In this state,
// depth == iter->depth, at_start == false, and error == SUCCESS.
// depth == iter.depth, at_start == false, and error == SUCCESS.
// - Unfinished Business: When we hand an array/object to the user which they do not fully
// iterate over, we need to finish that iteration by skipping child values until we reach the
// Next state. In this state, depth > iter->depth, at_start == false, and error == SUCCESS.
// Next state. In this state, depth > iter.depth, at_start == false, and error == SUCCESS.
//
// ## Error States
//
// In error states, we will yield exactly one more value before stopping. iter->depth == depth
// In error states, we will yield exactly one more value before stopping. iter.depth == depth
// and at_start is always false. We decrement after yielding the error, moving to the Finished
// state.
//
// - Chained Error: When the object iterator is part of an error chain--for example, in
// `for (auto tweet : doc["tweets"])`, where the tweet field may be missing or not be an
// object--we yield that error in the loop, exactly once. In this state, error != SUCCESS and
// iter->depth == depth, and at_start == false. We decrement depth when we yield the error.
// iter.depth == depth, and at_start == false. We decrement depth when we yield the error.
// - Missing Comma Error: When the iterator ++ method discovers there is no comma between fields,
// we flag that as an error and treat it exactly the same as a Chained Error. In this state,
// error == TAPE_ERROR, iter->depth == depth, and at_start == false.
// error == TAPE_ERROR, iter.depth == depth, and at_start == false.
//
// Errors that occur while reading a field to give to the user (such as when the key is not a
// string or the field is missing a colon) are yielded immediately. Depth is then decremented,
@ -37,86 +37,48 @@ namespace ondemand {
//
// ## Terminal State
//
// The terminal state has iter->depth < depth. at_start is always false.
// The terminal state has iter.depth < depth. at_start is always false.
//
// - Finished: When we have reached a }, we are finished. We signal this by decrementing depth.
// In this state, iter->depth < depth, at_start == false, and error == SUCCESS.
// In this state, iter.depth < depth, at_start == false, and error == SUCCESS.
//
simdjson_really_inline object::object(json_iterator_ref &&_iter) noexcept
: iter{std::forward<json_iterator_ref>(_iter)},
at_start{iter.is_alive()}
{
}
simdjson_really_inline error_code object::find_field(const std::string_view key) noexcept {
if (!iter.is_alive()) { return NO_SUCH_FIELD; }
// Unless this is the first field, we need to advance past the , and check for }
error_code error;
bool has_value;
if (at_start) {
at_start = false;
has_value = true;
} else {
if ((error = iter.finish_child() )) { iter.abandon(); return error; }
if ((error = iter->has_next_field().get(has_value) )) { iter.abandon(); return error; }
}
while (has_value) {
// Get the key
raw_json_string actual_key;
if ((error = iter->field_key().get(actual_key) )) { iter.abandon(); return error; };
if ((error = iter->field_value() )) { iter.abandon(); return error; }
// Check if it matches
if (actual_key == key) {
logger::log_event(*iter, "match", key, -2);
return SUCCESS;
}
logger::log_event(*iter, "no match", key, -2);
SIMDJSON_TRY( iter->skip() ); // Skip the value entirely
if ((error = iter->has_next_field().get(has_value) )) { iter.abandon(); return error; }
}
// If the loop ended, we're out of fields to look at.
iter.finished_container();
return NO_SUCH_FIELD;
simdjson_warn_unused simdjson_really_inline error_code object::find_field_raw(const std::string_view key) noexcept {
return iter.find_field_raw(key);
}
simdjson_really_inline simdjson_result<value> object::operator[](const std::string_view key) & noexcept {
SIMDJSON_TRY( find_field(key) );
return value::start(iter.borrow());
SIMDJSON_TRY( find_field_raw(key) );
return value(iter.iter.child());
}
simdjson_really_inline simdjson_result<value> object::operator[](const std::string_view key) && noexcept {
SIMDJSON_TRY( find_field(key) );
return value::start(iter.borrow());
SIMDJSON_TRY( find_field_raw(key) );
return value(iter.iter.child());
}
simdjson_really_inline simdjson_result<object> object::start(json_iterator_ref &&iter) noexcept {
bool has_value;
SIMDJSON_TRY( iter->start_object().get(has_value) );
if (has_value) { iter.started_container(); } else { iter.abandon(); }
return object(std::forward<json_iterator_ref>(iter));
simdjson_really_inline simdjson_result<object> object::start(value_iterator &iter) noexcept {
simdjson_unused bool has_value;
SIMDJSON_TRY( iter.start_object().get(has_value) );
return object(iter);
}
simdjson_really_inline simdjson_result<object> object::start(const uint8_t *json, json_iterator_ref &&iter) noexcept {
bool has_value;
SIMDJSON_TRY( iter->start_object(json).get(has_value) );
if (has_value) { iter.started_container(); } else { iter.abandon(); }
return object(std::forward<json_iterator_ref>(iter));
simdjson_really_inline simdjson_result<object> object::try_start(value_iterator &iter) noexcept {
simdjson_unused bool has_value;
SIMDJSON_TRY( iter.try_start_object().get(has_value) );
return object(iter);
}
simdjson_really_inline object object::started(json_iterator_ref &&iter) noexcept {
if (iter->started_object()) { iter.started_container(); } else { iter.abandon(); }
return object(std::forward<json_iterator_ref>(iter));
simdjson_really_inline object object::started(value_iterator &iter) noexcept {
simdjson_unused bool has_value = iter.started_object();
return iter;
}
simdjson_really_inline object::object(const value_iterator &_iter) noexcept
: iter{_iter}
{
}
simdjson_really_inline object_iterator object::begin() noexcept {
if (at_start) {
iter.assert_is_active();
} else {
iter.assert_is_not_active();
}
at_start = false;
SIMDJSON_ASSUME( iter.at_start || !iter.iter.is_open() );
return iter;
}
simdjson_really_inline object_iterator object::end() noexcept {

25
include/simdjson/generic/ondemand/object.h
View File

@ -33,9 +33,9 @@ protected:
* @param doc The document containing the object. The iterator must be just after the opening `{`.
* @param error If this is not SUCCESS, creates an error chained object.
*/
static simdjson_really_inline simdjson_result<object> start(json_iterator_ref &&iter) noexcept;
static simdjson_really_inline simdjson_result<object> start(const uint8_t *json, json_iterator_ref &&iter) noexcept;
static simdjson_really_inline object started(json_iterator_ref &&iter) noexcept;
static simdjson_really_inline simdjson_result<object> start(value_iterator &iter) noexcept;
static simdjson_really_inline simdjson_result<object> try_start(value_iterator &iter) noexcept;
static simdjson_really_inline object started(value_iterator &iter) noexcept;
/**
* Internal object creation. Call object::begin(doc) instead of this.
@ -43,24 +43,11 @@ protected:
* @param doc The document containing the object. doc->depth must already be incremented to
* reflect the object's depth. The iterator must be just after the opening `{`.
*/
simdjson_really_inline object(json_iterator_ref &&_iter) noexcept;
simdjson_really_inline object(const value_iterator &_iter) noexcept;
simdjson_really_inline error_code find_field(const std::string_view key) noexcept;
simdjson_warn_unused simdjson_really_inline error_code find_field_raw(const std::string_view key) noexcept;
/**
* Document containing the primary iterator.
*
* PERF NOTE: expected to be elided in favor of the parent document: this is set when the object
* is first used, and never changes afterwards.
*/
json_iterator_ref iter{};
/**
* Whether we are at the start.
*
* PERF NOTE: this should be elided into inline control flow: it is only used for the first []
* or * call, and SSA optimizers commonly do first-iteration loop optimization.
*/
bool at_start{};
object_iterator iter{};
friend class value;
friend class document;

101
include/simdjson/generic/ondemand/object_iterator-inl.h
View File

@ -6,38 +6,113 @@ namespace ondemand {
// object_iterator
//
simdjson_really_inline object_iterator::object_iterator(json_iterator_ref &_iter) noexcept : iter{&_iter} {}
simdjson_really_inline object_iterator::object_iterator(const value_iterator &_iter) noexcept
: iter{_iter},
at_start{true}
{}
simdjson_really_inline simdjson_result<field> object_iterator::operator*() noexcept {
error_code error = (*iter)->error();
if (error) { iter->abandon(); return error; }
auto result = field::start(*iter);
error_code error = iter.error();
if (error) { iter.abandon(); return error; }
auto result = field::start(iter);
// TODO this is a safety rail ... users should exit loops as soon as they receive an error.
// Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free.
if (result.error()) { iter->abandon(); }
if (result.error()) { iter.abandon(); }
return result;
}
simdjson_really_inline bool object_iterator::operator==(const object_iterator &other) const noexcept {
return !(*this != other);
}
simdjson_really_inline bool object_iterator::operator!=(const object_iterator &) const noexcept {
return iter->is_alive();
return iter.is_open();
}
simdjson_really_inline object_iterator &object_iterator::operator++() noexcept {
// TODO this is a safety rail ... users should exit loops as soon as they receive an error.
// Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free.
if (!iter->is_alive()) { return *this; } // Iterator will be released if there is an error
if (!iter.is_open()) { return *this; } // Iterator will be released if there is an error
auto error = iter->finish_child();
if (error) { return *this; }
simdjson_unused error_code error;
if ((error = iter.finish_child() )) { return *this; }
bool has_value;
error = (*iter)->has_next_field().get(has_value);
if (error) { return *this; }
if (!has_value) { iter->finished_container(); }
simdjson_unused bool has_value;
if ((error = iter.has_next_field().get(has_value) )) { return *this; };
return *this;
}
//
// ### Live States
//
// While iterating or looking up values, depth >= iter.depth. at_start may vary. Error is
// always SUCCESS:
//
// - Start: This is the state when the object is first found and the iterator is just past the {.
// In this state, at_start == true.
// - Next: After we hand a scalar value to the user, or an array/object which they then fully
// iterate over, the iterator is at the , or } before the next value. In this state,
// depth == iter.depth, at_start == false, and error == SUCCESS.
// - Unfinished Business: When we hand an array/object to the user which they do not fully
// iterate over, we need to finish that iteration by skipping child values until we reach the
// Next state. In this state, depth > iter.depth, at_start == false, and error == SUCCESS.
//
// ## Error States
//
// In error states, we will yield exactly one more value before stopping. iter.depth == depth
// and at_start is always false. We decrement after yielding the error, moving to the Finished
// state.
//
// - Chained Error: When the object iterator is part of an error chain--for example, in
// `for (auto tweet : doc["tweets"])`, where the tweet field may be missing or not be an
// object--we yield that error in the loop, exactly once. In this state, error != SUCCESS and
// iter.depth == depth, and at_start == false. We decrement depth when we yield the error.
// - Missing Comma Error: When the iterator ++ method discovers there is no comma between fields,
// we flag that as an error and treat it exactly the same as a Chained Error. In this state,
// error == TAPE_ERROR, iter.depth == depth, and at_start == false.
//
// Errors that occur while reading a field to give to the user (such as when the key is not a
// string or the field is missing a colon) are yielded immediately. Depth is then decremented,
// moving to the Finished state without transitioning through an Error state at all.
//
// ## Terminal State
//
// The terminal state has iter.depth < depth. at_start is always false.
//
// - Finished: When we have reached a }, we are finished. We signal this by decrementing depth.
// In this state, iter.depth < depth, at_start == false, and error == SUCCESS.
//
simdjson_warn_unused simdjson_really_inline error_code object_iterator::find_field_raw(const std::string_view key) noexcept {
if (!iter.is_open()) { return NO_SUCH_FIELD; }
// Unless this is the first field, we need to advance past the , and check for }
error_code error;
bool has_value;
if (at_start) {
at_start = false;
has_value = true;
} else {
if ((error = iter.finish_child() )) { iter.abandon(); return error; }
if ((error = iter.has_next_field().get(has_value) )) { iter.abandon(); return error; }
}
while (has_value) {
// Get the key
raw_json_string actual_key;
if ((error = iter.field_key().get(actual_key) )) { iter.abandon(); return error; };
if ((error = iter.field_value() )) { iter.abandon(); return error; }
// Check if it matches
if (actual_key == key) {
logger::log_event(iter, "match", key, -2);
return SUCCESS;
}
logger::log_event(iter, "no match", key, -2);
SIMDJSON_TRY( iter.skip_child() ); // Skip the value entirely
if ((error = iter.has_next_field().get(has_value) )) { iter.abandon(); return error; }
}
// If the loop ended, we're out of fields to look at.
return NO_SUCH_FIELD;
}
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson

26
include/simdjson/generic/ondemand/object_iterator.h
View File

@ -31,9 +31,31 @@ public:
simdjson_really_inline bool operator!=(const object_iterator &) const noexcept;
// Checks for ']' and ','
simdjson_really_inline object_iterator &operator++() noexcept;
/**
* Find the field with the given key. May be used in place of ++.
*/
simdjson_warn_unused simdjson_really_inline error_code find_field_raw(const std::string_view key) noexcept;
private:
json_iterator_ref *iter{};
simdjson_really_inline object_iterator(json_iterator_ref &iter) noexcept;
/**
* The underlying JSON iterator.
*
* PERF NOTE: expected to be elided in favor of the parent document: this is set when the object
* is first used, and never changes afterwards.
*/
value_iterator iter{};
/**
* Whether we are at the start.
*
* PERF NOTE: this should be elided into inline control flow: it is only used for the first []
* or * call, and SSA optimizers commonly do first-iteration loop optimization.
*
* SAFETY: this is not safe; the object_iterator can be copied freely, so the state CAN be lost.
*/
bool at_start{};
simdjson_really_inline object_iterator(const value_iterator &iter) noexcept;
friend struct simdjson_result<object_iterator>;
friend class object;
};

2
include/simdjson/generic/ondemand/raw_json_string-inl.h
View File

@ -14,7 +14,7 @@ simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> ra
}
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter) const noexcept {
return unescape(iter.current_string_buf_loc);
return unescape(iter.string_buf_loc());
}
simdjson_unused simdjson_really_inline bool operator==(const raw_json_string &a, std::string_view b) noexcept {

8
include/simdjson/generic/ondemand/token_iterator.h
View File

@ -16,11 +16,10 @@ public:
* Exists so you can declare a variable and later assign to it before use.
*/
simdjson_really_inline token_iterator() noexcept = default;
simdjson_really_inline token_iterator(token_iterator &&other) noexcept = default;
simdjson_really_inline token_iterator &operator=(token_iterator &&other) noexcept = default;
simdjson_really_inline token_iterator(const token_iterator &other) noexcept = delete;
simdjson_really_inline token_iterator &operator=(const token_iterator &other) noexcept = delete;
simdjson_really_inline token_iterator(const token_iterator &other) noexcept = default;
simdjson_really_inline token_iterator &operator=(const token_iterator &other) noexcept = default;
/**
* Get the JSON text for a given token (relative).
@ -76,6 +75,9 @@ protected:
const uint8_t *buf{};
const uint32_t *index{};
friend class json_iterator;
friend simdjson_really_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
};
} // namespace ondemand

164
include/simdjson/generic/ondemand/value-inl.h
View File

@ -2,130 +2,63 @@ namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
simdjson_really_inline value::value(json_iterator_ref && _iter, const uint8_t *_json) noexcept
: iter{std::forward<json_iterator_ref>(_iter)},
json{_json}
simdjson_really_inline value::value(const value_iterator &_iter) noexcept
: iter{_iter}
{
iter.assert_is_active();
SIMDJSON_ASSUME(json);
}
simdjson_really_inline value::value(value &&other) noexcept
: iter{std::move(other.iter)},
json{other.json} {
other.json = nullptr;
}
simdjson_really_inline value &value::operator=(value && other) noexcept {
iter = std::move(other.iter);
json = other.json;
other.json = nullptr;
return *this;
}
simdjson_really_inline value::~value() noexcept {
// If the user didn't actually use the value, we need to check if it's an array/object and bump
// depth so that the array/object iteration routines will work correctly.
// PERF TODO this better be elided entirely when people actually use the value. Don't care if it
// gets bumped on the error path unless that's costing us something important.
if (json) {
switch (*json) {
case '[': {
logger::log_start_value(*iter, "unused array");
if (iter->started_array()) { iter.started_container(); }
break;
}
case '{': {
logger::log_start_value(*iter, "unused object");
if (iter->started_object()) { iter.started_container(); }
break;
}
default: logger::log_value(*iter, "unused");
}
}
}
simdjson_really_inline value value::start(json_iterator_ref &&iter) noexcept {
return { std::forward<json_iterator_ref>(iter), iter->advance() };
}
simdjson_really_inline const uint8_t *value::consume() noexcept {
auto old_json = json;
json = nullptr;
return old_json;
}
simdjson_really_inline json_iterator_ref value::consume_container() noexcept {
consume();
return std::move(iter);
}
template<typename T>
simdjson_really_inline simdjson_result<T> value::consume_if_success(simdjson_result<T> &&result) noexcept {
if (!result.error()) { consume(); }
return std::forward<simdjson_result<T>>(result);
}
simdjson_really_inline error_code value::consume_if_success(error_code error) noexcept {
if (!error) { consume(); }
return error;
}
simdjson_really_inline simdjson_result<array> value::get_array() & noexcept {
if (*json != '[') { return INCORRECT_TYPE; }
return array::started(consume_container());
}
simdjson_really_inline simdjson_result<array> value::get_array() && noexcept {
return array::start(json, consume_container());
return array::start(iter);
}
simdjson_really_inline simdjson_result<object> value::get_object() & noexcept {
if (*json != '{') { return INCORRECT_TYPE; }
return object::started(consume_container());
simdjson_really_inline simdjson_result<array> value::get_array() & noexcept {
return array::try_start(iter);
}
simdjson_really_inline simdjson_result<object> value::get_object() && noexcept {
return object::start(json, consume_container());
return object::start(iter);
}
simdjson_really_inline simdjson_result<object> value::get_object() & noexcept {
return object::try_start(iter);
}
simdjson_really_inline simdjson_result<raw_json_string> value::get_raw_json_string() && noexcept {
return iter->consume_raw_json_string();
return iter.require_raw_json_string();
}
simdjson_really_inline simdjson_result<raw_json_string> value::get_raw_json_string() & noexcept {
return consume_if_success( iter->parse_raw_json_string(json) );
return iter.try_get_raw_json_string();
}
simdjson_really_inline simdjson_result<std::string_view> value::get_string() && noexcept {
auto result = iter->parse_string(json);
consume();
return result;
return iter.require_string();
}
simdjson_really_inline simdjson_result<std::string_view> value::get_string() & noexcept {
return consume_if_success( iter->parse_string(json) );
return iter.try_get_string();
}
simdjson_really_inline simdjson_result<double> value::get_double() && noexcept {
return iter->parse_double(consume());
return iter.require_double();
}
simdjson_really_inline simdjson_result<double> value::get_double() & noexcept {
return consume_if_success( iter->parse_double(json) );
return iter.try_get_double();
}
simdjson_really_inline simdjson_result<uint64_t> value::get_uint64() && noexcept {
return iter->parse_uint64(consume());
return iter.require_uint64();
}
simdjson_really_inline simdjson_result<uint64_t> value::get_uint64() & noexcept {
return consume_if_success( iter->parse_uint64(json) );
return iter.try_get_uint64();
}
simdjson_really_inline simdjson_result<int64_t> value::get_int64() && noexcept {
return iter->parse_int64(consume());
return iter.require_int64();
}
simdjson_really_inline simdjson_result<int64_t> value::get_int64() & noexcept {
return consume_if_success( iter->parse_int64(json) );
return iter.try_get_int64();
}
simdjson_really_inline simdjson_result<bool> value::get_bool() && noexcept {
return iter->parse_bool(consume());
return iter.require_bool();
}
simdjson_really_inline simdjson_result<bool> value::get_bool() & noexcept {
return consume_if_success( iter->parse_bool(json) );
return iter.try_get_bool();
}
simdjson_really_inline bool value::is_null() && noexcept {
return iter->is_null(consume());
return iter.require_null();
}
simdjson_really_inline bool value::is_null() & noexcept {
if (!iter->is_null(json)) { return false; }
consume();
return true;
return iter.is_null();
}
template<> simdjson_really_inline simdjson_result<array> value::get() & noexcept { return get_array(); }
@ -205,46 +138,21 @@ simdjson_really_inline value::operator bool() & noexcept(false) {
}
#endif
simdjson_really_inline simdjson_result<array_iterator<value>> value::begin() & noexcept {
return consume_if_success( array_iterator<value>::start(*this, json) );
simdjson_really_inline simdjson_result<array_iterator> value::begin() & noexcept {
return get_array().begin();
}
simdjson_really_inline simdjson_result<array_iterator<value>> value::end() & noexcept {
simdjson_really_inline simdjson_result<array_iterator> value::end() & noexcept {
return {};
}
simdjson_really_inline void value::log_value(const char *type) const noexcept {
char json_char[]{char(json[0]), '\0'};
logger::log_value(*iter, type, json_char);
}
simdjson_really_inline void value::log_error(const char *message) const noexcept {
char json_char[]{char(json[0]), '\0'};
logger::log_error(*iter, message, json_char);
}
//
// For array_iterator
//
simdjson_really_inline json_iterator &value::get_iterator() noexcept {
return *iter;
}
simdjson_really_inline json_iterator_ref value::borrow_iterator_child() noexcept {
return iter.borrow();
}
simdjson_really_inline bool value::is_iterator_alive() const noexcept {
return iter.is_alive();
}
simdjson_really_inline void value::start_iterator() noexcept {
iter.started_container();
}
simdjson_really_inline void value::finish_iterator() noexcept {
iter.finished_container();
}
simdjson_really_inline void value::abandon_iterator() noexcept {
iter.abandon();
}
simdjson_warn_unused simdjson_really_inline error_code value::finish_iterator_child() noexcept {
return iter.finish_child();
}
// simdjson_really_inline void value::log_value(const char *type) const noexcept {
// char json_char[]{char(json[0]), '\0'};
// logger::log_value(*iter, type, json_char);
// }
// simdjson_really_inline void value::log_error(const char *message) const noexcept {
// char json_char[]{char(json[0]), '\0'};
// logger::log_error(*iter, message, json_char);
// }
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
@ -267,11 +175,11 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>
{
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::value>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::begin() & noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::begin() & noexcept {
if (error()) { return error(); }
return first.begin();
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::value>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::end() & noexcept {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::end() & noexcept {
if (error()) { return error(); }
return {};
}

59
include/simdjson/generic/ondemand/value.h
View File

@ -22,15 +22,11 @@ public:
*/
simdjson_really_inline value() noexcept = default;
simdjson_really_inline value(value &&other) noexcept;
simdjson_really_inline value &operator=(value && other) noexcept;
simdjson_really_inline value(const value &) noexcept = delete;
simdjson_really_inline value &operator=(const value &) noexcept = delete;
/**
* Skips the value if the value was not successfully parsed or used.
*/
simdjson_really_inline ~value() noexcept;
simdjson_really_inline value(value &&other) noexcept = default;
simdjson_really_inline value &operator=(value && other) noexcept = default;
simdjson_really_inline value(const value &) noexcept = default;
simdjson_really_inline value &operator=(const value &) noexcept = default;
simdjson_really_inline ~value() noexcept = default;
/**
* Get this value as the given type.
@ -246,13 +242,13 @@ public:
*
* @returns INCORRECT_TYPE If the JSON value is not an array.
*/
simdjson_really_inline simdjson_result<array_iterator<value>> begin() & noexcept;
simdjson_really_inline simdjson_result<array_iterator> begin() & noexcept;
/**
* Sentinel representing the end of the array.
*
* Part of the std::iterable interface.
*/
simdjson_really_inline simdjson_result<array_iterator<value>> end() & noexcept;
simdjson_really_inline simdjson_result<array_iterator> end() & noexcept;
protected:
/**
@ -260,47 +256,20 @@ protected:
*
* Use value::read() instead of this.
*/
simdjson_really_inline value(json_iterator_ref &&iter, const uint8_t *json) noexcept;
/**
* Read a value.
*
* If the value is an array or object, only the opening brace will be consumed.
*
* @param doc The document containing the value. Iterator must be at the value start position.
*/
static simdjson_really_inline value start(json_iterator_ref &&iter) noexcept;
simdjson_really_inline value(const value_iterator &iter) noexcept;
/**
* Skip this value, allowing iteration to continue.
*/
simdjson_really_inline void skip() noexcept;
simdjson_really_inline void log_value(const char *type) const noexcept;
simdjson_really_inline void log_error(const char *message) const noexcept;
// simdjson_really_inline void log_value(const char *type) const noexcept;
// simdjson_really_inline void log_error(const char *message) const noexcept;
//
// For array_iterator
//
simdjson_really_inline json_iterator &get_iterator() noexcept;
simdjson_really_inline json_iterator_ref borrow_iterator_child() noexcept;
simdjson_really_inline bool is_iterator_alive() const noexcept;
simdjson_really_inline void start_iterator() noexcept;
simdjson_really_inline void finish_iterator() noexcept;
simdjson_really_inline void abandon_iterator() noexcept;
simdjson_warn_unused simdjson_really_inline error_code finish_iterator_child() noexcept;
simdjson_really_inline const uint8_t *consume() noexcept;
simdjson_really_inline json_iterator_ref consume_container() noexcept;
template<typename T>
simdjson_really_inline simdjson_result<T> consume_if_success(simdjson_result<T> &&result) noexcept;
simdjson_warn_unused simdjson_really_inline error_code consume_if_success(error_code error) noexcept;
json_iterator_ref iter{};
const uint8_t *json{}; // The JSON text of the value
value_iterator iter{};
friend class document;
template<typename T> friend class array_iterator;
friend class array_iterator;
friend class field;
friend class object;
friend struct simdjson_result<value>;
@ -376,8 +345,8 @@ public:
simdjson_really_inline operator bool() & noexcept(false);
#endif
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::value>> begin() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::value>> end() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
};
} // namespace simdjson

430
include/simdjson/generic/ondemand/value_iterator-inl.h Normal file
View File

@ -0,0 +1,430 @@
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
simdjson_really_inline value_iterator::value_iterator(json_iterator *json_iter, depth_t depth) noexcept
: _json_iter{json_iter},
_depth{depth}
{
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::start_object() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0 );
if (*_json_iter->advance() != '{') { logger::log_error(*_json_iter, "Not an object"); return INCORRECT_TYPE; }
return started_object();
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::try_start_object() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0 );
if (*_json_iter->peek() != '{') { logger::log_error(*_json_iter, "Not an object"); return INCORRECT_TYPE; }
_json_iter->advance();
return started_object();
}
simdjson_warn_unused simdjson_really_inline bool value_iterator::started_object() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0 );
if (*_json_iter->peek() == '}') {
logger::log_value(*_json_iter, "empty object");
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return false;
}
_json_iter->descend_to(depth()+1);
logger::log_start_value(*_json_iter, "object");
return true;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_field() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() );
switch (*_json_iter->advance()) {
case '}':
logger::log_end_value(*_json_iter, "object");
_json_iter->ascend_to(depth()-1);
return false;
case ',':
_json_iter->descend_to(depth()+1);
return true;
default:
return _json_iter->report_error(TAPE_ERROR, "Missing comma between object fields");
}
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::find_field_raw(const char *key) noexcept {
// We assume we are sitting at a key: at "key": <value>
SIMDJSON_ASSUME( _json_iter->depth() == depth()+1 );
bool has_next;
do {
// Get the key
raw_json_string actual_key;
SIMDJSON_TRY( require_raw_json_string().get(actual_key) );
if (*_json_iter->advance() != ':') { return _json_iter->report_error(TAPE_ERROR, "Missing colon in object field"); }
// Check if the key matches, and return if so
if (actual_key == key) {
logger::log_event(*_json_iter, "match", key);
return true;
}
// Skip the value so we can look at the next key
logger::log_event(*_json_iter, "non-match", key);
SIMDJSON_TRY( skip_child() );
// Check whether the next token is , or }
SIMDJSON_TRY( has_next_field().get(has_next) );
} while (has_next);
logger::log_event(*_json_iter, "no matches", key);
return false;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth()+1 );
const uint8_t *key = _json_iter->advance();
if (*(key++) != '"') { return _json_iter->report_error(TAPE_ERROR, "Object key is not a string"); }
return raw_json_string(key);
}
simdjson_warn_unused simdjson_really_inline error_code value_iterator::field_value() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth()+1 );
if (*_json_iter->advance() != ':') { return _json_iter->report_error(TAPE_ERROR, "Missing colon in object field"); }
return SUCCESS;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::start_array() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0);
if (*_json_iter->advance() != '[') { logger::log_error(*_json_iter, "Not an array"); return INCORRECT_TYPE; }
return started_array();
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::try_start_array() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0);
if (*_json_iter->peek() != '[') { logger::log_error(*_json_iter, "Not an array"); return INCORRECT_TYPE; }
_json_iter->advance();
return started_array();
}
simdjson_warn_unused simdjson_really_inline bool value_iterator::started_array() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0 );
if (*_json_iter->peek() == ']') {
logger::log_value(*_json_iter, "empty array");
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return false;
}
logger::log_start_value(*_json_iter, "array");
_json_iter->descend_to(depth()+1);
return true;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_element() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() );
switch (*_json_iter->advance()) {
case ']':
logger::log_end_value(*_json_iter, "array");
_json_iter->ascend_to(depth()-1);
return false;
case ',':
_json_iter->descend_to(depth()+1);
return true;
default:
return _json_iter->report_error(TAPE_ERROR, "Missing comma between array elements");
}
}
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::try_get_string() noexcept {
return try_get_raw_json_string().unescape(_json_iter->string_buf_loc());
}
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::require_string() noexcept {
return require_raw_json_string().unescape(_json_iter->string_buf_loc());
}
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::try_get_raw_json_string() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0 );
logger::log_value(*_json_iter, "string", "");
auto json = _json_iter->peek();
if (*json != '"') { logger::log_error(*_json_iter, "Not a string"); return INCORRECT_TYPE; }
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return raw_json_string(json+1);
}
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::require_raw_json_string() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 0 );
logger::log_value(*_json_iter, "string", "");
auto json = _json_iter->advance();
if (*json != '"') { logger::log_error(*_json_iter, "Not a string"); return INCORRECT_TYPE; }
_json_iter->ascend_to(depth()-1);
return raw_json_string(json+1);
}
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::try_get_uint64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
logger::log_value(*_json_iter, "uint64", "");
uint64_t result;
SIMDJSON_TRY( numberparsing::parse_unsigned(_json_iter->peek()).get(result) );
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::require_uint64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
logger::log_value(*_json_iter, "uint64", "");
_json_iter->ascend_to(depth()-1);
return numberparsing::parse_unsigned(_json_iter->advance());
}
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::try_get_int64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
logger::log_value(*_json_iter, "int64", "");
int64_t result;
SIMDJSON_TRY( numberparsing::parse_integer(_json_iter->peek()).get(result) );
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::require_int64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
logger::log_value(*_json_iter, "int64", "");
_json_iter->ascend_to(depth()-1);
return numberparsing::parse_integer(_json_iter->advance());
}
simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::try_get_double() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
logger::log_value(*_json_iter, "double", "");
double result;
SIMDJSON_TRY( numberparsing::parse_double(_json_iter->peek()).get(result) );
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::require_double() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
logger::log_value(*_json_iter, "double", "");
_json_iter->ascend_to(depth()-1);
return numberparsing::parse_double(_json_iter->advance());
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::parse_bool(const uint8_t *json) const noexcept {
logger::log_value(*_json_iter, "bool", "");
auto not_true = atomparsing::str4ncmp(json, "true");
auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
if (error) { logger::log_error(*_json_iter, "Not a boolean"); return INCORRECT_TYPE; }
return simdjson_result<bool>(!not_true);
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::try_get_bool() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
bool result;
SIMDJSON_TRY( parse_bool(_json_iter->peek()).get(result) );
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::require_bool() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
_json_iter->ascend_to(depth()-1);
return parse_bool(_json_iter->advance());
}
simdjson_really_inline bool value_iterator::is_null(const uint8_t *json) const noexcept {
if (!atomparsing::str4ncmp(json, "null")) {
logger::log_value(*_json_iter, "null", "");
return true;
}
return false;
}
simdjson_really_inline bool value_iterator::is_null() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
if (is_null(_json_iter->peek())) {
_json_iter->advance();
_json_iter->ascend_to(depth()-1);
return true;
}
return false;
}
simdjson_really_inline bool value_iterator::require_null() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() > 1 );
_json_iter->ascend_to(depth()-1);
return is_null(_json_iter->advance());
}
constexpr const uint32_t MAX_INT_LENGTH = 1024;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::parse_root_uint64(const uint8_t *json, uint32_t max_len) const noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { logger::log_error(*_json_iter, "Root number more than 20 characters"); return NUMBER_ERROR; }
logger::log_value(*_json_iter, "uint64", "");
auto result = numberparsing::parse_unsigned(tmpbuf);
if (result.error()) { logger::log_error(*_json_iter, "Error parsing unsigned integer"); }
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::try_get_root_uint64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
uint64_t result;
SIMDJSON_TRY( parse_root_uint64(_json_iter->peek(), _json_iter->peek_length()).get(result) );
_json_iter->advance();
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::require_root_uint64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
auto max_len = _json_iter->peek_length();
return parse_root_uint64(_json_iter->advance(), max_len);
}
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::parse_root_int64(const uint8_t *json, uint32_t max_len) const noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { logger::log_error(*_json_iter, "Root number more than 20 characters"); return NUMBER_ERROR; }
logger::log_value(*_json_iter, "int64", "");
auto result = numberparsing::parse_integer(tmpbuf);
if (result.error()) { logger::log_error(*_json_iter, "Error parsing integer"); }
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::try_get_root_int64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
int64_t result;
SIMDJSON_TRY( parse_root_int64(_json_iter->peek(), _json_iter->peek_length()).get(result) );
_json_iter->advance();
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::require_root_int64() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
auto max_len = _json_iter->peek_length();
return parse_root_int64(_json_iter->advance(), max_len);
}
simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::parse_root_double(const uint8_t *json, uint32_t max_len) const noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest number: -0.<fraction>e-308.
uint8_t tmpbuf[1074+8+1];
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { logger::log_error(*_json_iter, "Root number more than 1082 characters"); return NUMBER_ERROR; }
logger::log_value(*_json_iter, "double", "");
auto result = numberparsing::parse_double(tmpbuf);
if (result.error()) { logger::log_error(*_json_iter, "Error parsing double"); }
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::try_get_root_double() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
double result;
SIMDJSON_TRY( parse_root_double(_json_iter->peek(), _json_iter->peek_length()).get(result) );
_json_iter->advance();
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::require_root_double() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
auto max_len = _json_iter->peek_length();
return parse_root_double(_json_iter->advance(), max_len);
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::parse_root_bool(const uint8_t *json, uint32_t max_len) const noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
uint8_t tmpbuf[5+1];
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { logger::log_error(*_json_iter, "Not a boolean"); return INCORRECT_TYPE; }
return parse_bool(tmpbuf);
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::try_get_root_bool() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
bool result;
SIMDJSON_TRY( parse_root_bool(_json_iter->peek(), _json_iter->peek_length()).get(result) );
_json_iter->advance();
return result;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::require_root_bool() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
auto max_len = _json_iter->peek_length();
return parse_root_bool(_json_iter->advance(), max_len);
}
simdjson_really_inline bool value_iterator::is_root_null(const uint8_t *json, uint32_t max_len) const noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
uint8_t tmpbuf[4+1];
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { return false; }
return is_null(tmpbuf);
}
simdjson_really_inline bool value_iterator::is_root_null() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
if (!is_root_null(_json_iter->peek(), _json_iter->peek_length())) { return false; }
_json_iter->advance();
return true;
}
simdjson_really_inline bool value_iterator::require_root_null() noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth() && depth() == 1 );
auto max_len = _json_iter->peek_length();
return is_root_null(_json_iter->advance(), max_len);
}
simdjson_warn_unused simdjson_really_inline error_code value_iterator::skip_child() noexcept {
return _json_iter->skip_child(depth());
}
simdjson_warn_unused simdjson_really_inline error_code value_iterator::finish_child() noexcept {
return _json_iter->finish_child(depth());
}
simdjson_really_inline value_iterator value_iterator::child() const noexcept {
SIMDJSON_ASSUME( _json_iter->depth() == depth()+1 );
return { _json_iter, depth()+1 };
}
simdjson_really_inline bool value_iterator::is_open() const noexcept {
return _json_iter->depth() >= depth();
}
simdjson_really_inline void value_iterator::abandon() noexcept {
_json_iter->abandon();
}
simdjson_warn_unused simdjson_really_inline depth_t value_iterator::depth() const noexcept {
return _depth;
}
simdjson_warn_unused simdjson_really_inline error_code value_iterator::error() const noexcept {
return _json_iter->error;
}
simdjson_warn_unused simdjson_really_inline uint8_t *&value_iterator::string_buf_loc() noexcept {
return _json_iter->string_buf_loc();
}
simdjson_warn_unused simdjson_really_inline const json_iterator &value_iterator::json_iter() const noexcept {
return *_json_iter;
}
simdjson_warn_unused simdjson_really_inline json_iterator &value_iterator::json_iter() noexcept {
return *_json_iter;
}
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
namespace simdjson {
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator>::simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::value_iterator &&value) noexcept
: implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator>(value)) {}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator>::simdjson_result(error_code error) noexcept
: implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator>(error) {}
} // namespace simdjson

290
include/simdjson/generic/ondemand/value_iterator.h Normal file
View File

@ -0,0 +1,290 @@
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
class document;
class object;
class array;
class value;
class raw_json_string;
class parser;
/**
* Iterates through a single JSON value at a particular depth.
*
* Does not keep track of the type of value: provides methods for objects, arrays and scalars and expects
* the caller to call the right ones.
*
* @private This is not intended for external use.
*/
class value_iterator {
protected:
/** The underlying JSON iterator */
json_iterator *_json_iter{};
/** The depth of this value */
depth_t _depth{};
public:
simdjson_really_inline value_iterator() noexcept = default;
simdjson_really_inline value_iterator(value_iterator &&other) noexcept = default;
simdjson_really_inline value_iterator &operator=(value_iterator &&other) noexcept = default;
simdjson_really_inline value_iterator(const value_iterator &other) noexcept = default;
simdjson_really_inline value_iterator &operator=(const value_iterator &other) noexcept = default;
/**
* Denote that we're starting a document.
*/
simdjson_really_inline void start_document() noexcept;
/**
* Skips a non-iterated JSON value, whether it is a scalar, array or object.
*
* Optimized for scalars.
*/
simdjson_warn_unused simdjson_really_inline error_code skip_child() noexcept;
/**
* Skips a possibly-partially-iterated JSON value, whether it is a scalar, array or object.
*/
simdjson_warn_unused simdjson_really_inline error_code finish_child() noexcept;
/**
* Tell whether the iterator is at the EOF mark
*/
simdjson_really_inline bool at_eof() const noexcept;
/**
* Tell whether the value is open--if the value has not been used, or the array/object is still open.
*/
simdjson_really_inline bool is_open() const noexcept;
/**
* Abandon all iteration.
*/
simdjson_really_inline void abandon() noexcept;
/**
* Get the child value as a value_iterator.
*/
simdjson_really_inline value_iterator child_value() const noexcept;
/**
* Get the depth of this value.
*/
simdjson_really_inline depth_t depth() const noexcept;
/**
* @addtogroup object Object iteration
*
* Methods to iterate and find object fields. These methods generally *assume* the value is
* actually an object; the caller is responsible for keeping track of that fact.
*
* @{
*/
/**
* Start an object iteration.
*
* @returns Whether the object had any fields (returns false for empty).
* @error INCORRECT_TYPE if there is no opening {
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> start_object() noexcept;
/**
* Check for an opening { and start an object iteration.
*
* @returns Whether the object had any fields (returns false for empty).
* @error INCORRECT_TYPE if there is no opening {
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> try_start_object() noexcept;
/**
* Start an object iteration after the user has already checked and moved past the {.
*
* Does not move the iterator.
*
* @returns Whether the object had any fields (returns false for empty).
*/
simdjson_warn_unused simdjson_really_inline bool started_object() noexcept;
/**
* Moves to the next field in an object.
*
* Looks for , and }. If } is found, the object is finished and the iterator advances past it.
* Otherwise, it advances to the next value.
*
* @return whether there is another field in the object.
* @error TAPE_ERROR If there is a comma missing between fields.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> has_next_field() noexcept;
/**
* Get the current field's key.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> field_key() noexcept;
/**
* Pass the : in the field and move to its value.
*/
simdjson_warn_unused simdjson_really_inline error_code field_value() noexcept;
/**
* Find the next field with the given key.
*
* Assumes you have called next_field() or otherwise matched the previous value.
*
* This means the iterator must be sitting at the next key:
*
* ```
* { "a": 1, "b": 2 }
* ^
* ```
*
* Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to
* unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may
* fail to match some keys with escapes (\u, \n, etc.).
*/
simdjson_warn_unused simdjson_really_inline error_code find_field(const std::string_view key) noexcept;
/**
* Find the next field with the given key, *without* unescaping.
*
* Assumes you have called next_field() or otherwise matched the previous value.
*
* This means the iterator must be sitting at the next key:
*
* ```
* { "a": 1, "b": 2 }
* ^
* ```
*
* Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to
* unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may
* fail to match some keys with escapes (\u, \n, etc.).
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> find_field_raw(const char *key) noexcept;
/** @} */
/**
* @addtogroup array Array iteration
* Methods to iterate over array elements. These methods generally *assume* the value is actually
* an object; the caller is responsible for keeping track of that fact.
* @{
*/
/**
* Check for an opening [ and start an array iteration.
*
* @param json A pointer to the potential [.
* @returns Whether the array had any elements (returns false for empty).
* @error INCORRECT_TYPE If there is no [.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> start_array() noexcept;
/**
* Check for an opening [ and start an array iteration.
*
* @returns Whether the array had any elements (returns false for empty).
* @error INCORRECT_TYPE If there is no [.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> try_start_array() noexcept;
/**
* Start an array iteration after the user has already checked and moved past the [.
*
* Does not move the iterator.
*
* @returns Whether the array had any elements (returns false for empty).
*/
simdjson_warn_unused simdjson_really_inline bool started_array() noexcept;
/**
* Moves to the next element in an array.
*
* Looks for , and ]. If ] is found, the array is finished and the iterator advances past it.
* Otherwise, it advances to the next value.
*
* @return Whether there is another element in the array.
* @error TAPE_ERROR If there is a comma missing between elements.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> has_next_element() noexcept;
/**
* Get a child value iterator.
*/
simdjson_warn_unused simdjson_really_inline value_iterator child() const noexcept;
/** @} */
/**
* @defgroup scalar Scalar values
* @addtogroup scalar
* @{
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> try_get_string() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> require_string() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> try_get_raw_json_string() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> require_raw_json_string() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> try_get_uint64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> require_uint64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> try_get_int64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> require_int64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> try_get_double() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> require_double() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> try_get_bool() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> require_bool() noexcept;
simdjson_really_inline bool require_null() noexcept;
simdjson_really_inline bool is_null() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> try_get_root_uint64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> require_root_uint64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> try_get_root_int64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> require_root_int64() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> try_get_root_double() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<double> require_root_double() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> try_get_root_bool() noexcept;
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> require_root_bool() noexcept;
simdjson_really_inline bool require_root_null() noexcept;
simdjson_really_inline bool is_root_null() noexcept;
simdjson_really_inline error_code error() const noexcept;
simdjson_really_inline uint8_t *&string_buf_loc() noexcept;
simdjson_really_inline const json_iterator &json_iter() const noexcept;
simdjson_really_inline json_iterator &json_iter() noexcept;
/** @} */
protected:
simdjson_really_inline value_iterator(json_iterator *json_iter, depth_t depth) noexcept;
simdjson_really_inline bool is_null(const uint8_t *json) const noexcept;
simdjson_really_inline simdjson_result<bool> parse_bool(const uint8_t *json) const noexcept;
simdjson_really_inline bool is_root_null(const uint8_t *json, uint32_t max_len) const noexcept;
simdjson_really_inline simdjson_result<bool> parse_root_bool(const uint8_t *json, uint32_t max_len) const noexcept;
simdjson_really_inline simdjson_result<uint64_t> parse_root_uint64(const uint8_t *json, uint32_t max_len) const noexcept;
simdjson_really_inline simdjson_result<int64_t> parse_root_int64(const uint8_t *json, uint32_t max_len) const noexcept;
simdjson_really_inline simdjson_result<double> parse_root_double(const uint8_t *json, uint32_t max_len) const noexcept;
friend class document;
friend class object;
friend class array;
friend class value;
}; // value_iterator
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
namespace simdjson {
template<>
struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator> {
public:
simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::value_iterator &&value) noexcept; ///< @private
simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
simdjson_really_inline simdjson_result() noexcept = default;
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value_iterator> &&a) noexcept = default;
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
};
} // namespace simdjson

31
tests/ondemand/ondemand_basictests.cpp
View File

@ -1274,8 +1274,11 @@ namespace error_tests {
V actual;
auto actual_error = elem.get(actual);
if (count >= N) {
if (count >= (N+N2)) {
std::cerr << "FAIL: Extra error reported: " << actual_error << std::endl;
return false;
}
ASSERT_ERROR(actual_error, expected_error[count - N]);
ASSERT(count < (N+N2), "Extra error reported");
} else {
ASSERT_SUCCESS(actual_error);
ASSERT_EQUAL(actual, expected[count]);
@ -1320,8 +1323,8 @@ namespace error_tests {
TEST_START();
ONDEMAND_SUBTEST("missing comma", "[1 1]", assert_iterate(doc, { int64_t(1) }, { TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[1,,1]", assert_iterate(doc, { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,]", assert_iterate(doc, { NUMBER_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,,]", assert_iterate(doc, { NUMBER_ERROR, NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,]", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,,]", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
TEST_SUCCEED();
}
bool top_level_array_iterate_unclosed_error() {
@ -1330,7 +1333,7 @@ namespace error_tests {
ONDEMAND_SUBTEST("unclosed ", "[1 ", assert_iterate(doc, { int64_t(1) }, { TAPE_ERROR }));
// TODO These pass the user values that may run past the end of the buffer if they aren't careful
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed extra comma", "[,,", assert_iterate(doc, { NUMBER_ERROR, NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed extra comma", "[,,", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", "[1,", assert_iterate(doc, { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", "[1", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", "[", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
@ -1341,15 +1344,15 @@ namespace error_tests {
TEST_START();
ONDEMAND_SUBTEST("missing comma", R"({ "a": [1 1] })", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,1] })", assert_iterate(doc["a"], { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,] })", assert_iterate(doc["a"], { int64_t(1) }, { NUMBER_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,] })", assert_iterate(doc["a"], { NUMBER_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,,] })", assert_iterate(doc["a"], { NUMBER_ERROR, NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,] })", assert_iterate(doc["a"], { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,] })", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,,] })", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
TEST_SUCCEED();
}
bool array_iterate_unclosed_error() {
TEST_START();
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { NUMBER_ERROR, NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1 )", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR }));
// TODO These pass the user values that may run past the end of the buffer if they aren't careful
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
@ -1537,12 +1540,12 @@ int main(int argc, char *argv[]) {
std::cout << "Running basic tests." << std::endl;
if (
// parse_api_tests::run() &&
// dom_api_tests::run() &&
// twitter_tests::run() &&
// number_tests::run() &&
// ordering_tests::run() &&
// key_string_tests::run() &&
parse_api_tests::run() &&
dom_api_tests::run() &&
twitter_tests::run() &&
number_tests::run() &&
ordering_tests::run() &&
key_string_tests::run() &&
active_tests::run() &&
error_tests::run() &&
true