jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Allow use of document_stream with tie()

This commit is contained in:
John Keiser 2020-06-20 15:47:41 -07:00
parent 94440e0170
commit 9899e5021d
11 changed files with 145 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
benchmark/parse_stream.cpp
View File

@ -87,10 +87,15 @@ int main(int argc, char *argv[]) {
auto start = std::chrono::steady_clock::now();
count = 0;
for (auto result : parser.parse_many(p, i)) {
simdjson::dom::document_stream docs;
if ((error = parser.parse_many(p, i).get(docs))) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
for (auto result : docs) {
error = result.error();
if (error != simdjson::SUCCESS) {
std::wcerr << "Parsing failed with: " << error_message(error) << std::endl;
if (error) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
count++;
@ -134,10 +139,15 @@ int main(int argc, char *argv[]) {
auto start = std::chrono::steady_clock::now();
// This includes allocation of the parser
for (auto result : parser.parse_many(p, optimal_batch_size)) {
simdjson::dom::document_stream docs;
if ((error = parser.parse_many(p, optimal_batch_size).get(docs))) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
for (auto result : docs) {
error = result.error();
if (error != simdjson::SUCCESS) {
std::wcerr << "Parsing failed with: " << error_message(error) << std::endl;
if (error) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
}

34
include/simdjson/dom/document_stream.h
View File

@ -72,8 +72,20 @@ private:
*/
class document_stream {
public:
/**
* Construct an uninitialized document_stream.
*
* ```c++
* document_stream docs;
* error = parser.parse_many(json).get(docs);
* ```
*/
really_inline document_stream() noexcept;
/** Move one document_stream to another. */
really_inline document_stream(document_stream && other) noexcept = default;
really_inline document_stream(document_stream &&other) noexcept = default;
/** Move one document_stream to another. */
really_inline document_stream &operator=(document_stream &&other) noexcept = default;
really_inline ~document_stream() noexcept;
/**
@ -99,9 +111,8 @@ public:
*
* Gives the current index in the input document in bytes.
*
* auto stream = parser.parse_many(json,window);
* auto i = stream.begin();
* for(; i != stream.end(); ++i) {
* document_stream stream = parser.parse_many(json,window);
* for(auto i = stream.begin(); i != stream.end(); ++i) {
* auto doc = *i;
* size_t index = i.current_index();
* }
@ -135,9 +146,6 @@ private:
document_stream(document_stream &other) = delete; // Disallow copying
/** Construct an uninitialized document_stream **/
really_inline document_stream() noexcept;
/**
* Construct a document_stream. Does not allocate or parse anything until the iterator is
* used.
@ -193,13 +201,14 @@ private:
/** Pass the next batch through stage 1 with the given parser. */
inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept;
dom::parser &parser;
dom::parser *parser;
const uint8_t *buf;
const size_t len;
const size_t batch_size;
size_t batch_start{0};
size_t len;
size_t batch_size;
/** The error (or lack thereof) from the current document. */
error_code error;
size_t batch_start{0};
size_t doc_index{};
#ifdef SIMDJSON_THREADS_ENABLED
inline void load_from_stage1_thread() noexcept;
@ -223,10 +232,9 @@ private:
#endif // SIMDJSON_THREADS_ENABLED
friend class dom::parser;
friend class simdjson_result<dom::document_stream>;
friend class internal::simdjson_result_base<dom::document_stream>;
size_t doc_index{};
}; // class document_stream
} // namespace dom

19
include/simdjson/dom/element.h
View File

@ -243,25 +243,6 @@ public:
template<typename T>
inline void tie(T &value, error_code &error) && noexcept;
/**
* Get the value as the provided type (T).
*
* Supported types:
* - Boolean: bool
* - Number: double, uint64_t, int64_t
* - String: std::string_view, const char *
* - Array: dom::array
* - Object: dom::object
*
* @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object
*
* @param value The variable to set to the given type. value is undefined if there is an error.
*
* @returns true if the value was able to be set, false if there was an error.
*/
template<typename T>
WARN_UNUSED inline bool tie(T &value) && noexcept;
#if SIMDJSON_EXCEPTIONS
/**
* Read this element as a boolean.

2
include/simdjson/dom/parser.h
View File

@ -205,7 +205,7 @@ public:
* Parse a buffer containing many JSON documents.
*
* dom::parser parser;
* for (const element doc : parser.parse_many(buf, len)) {
* for (element doc : parser.parse_many(buf, len)) {
* cout << std::string(doc["title"]) << endl;
* }
*

32
include/simdjson/inline/document_stream.h
View File

@ -70,7 +70,7 @@ really_inline document_stream::document_stream(
size_t _len,
size_t _batch_size
) noexcept
: parser{_parser},
: parser{&_parser},
buf{_buf},
len{_len},
batch_size{_batch_size},
@ -83,7 +83,15 @@ really_inline document_stream::document_stream(
#endif
}
inline document_stream::~document_stream() noexcept {
really_inline document_stream::document_stream() noexcept
: parser{nullptr},
buf{nullptr},
len{0},
batch_size{0},
error{UNINITIALIZED} {
}
really_inline document_stream::~document_stream() noexcept {
}
really_inline document_stream::iterator document_stream::begin() noexcept {
@ -103,7 +111,7 @@ really_inline document_stream::iterator::iterator(document_stream& _stream, bool
really_inline simdjson_result<element> document_stream::iterator::operator*() noexcept {
// Once we have yielded any errors, we're finished.
if (stream.error) { finished = true; return stream.error; }
return stream.parser.doc.root();
return stream.parser->doc.root();
}
really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept {
@ -120,12 +128,12 @@ really_inline bool document_stream::iterator::operator!=(const document_stream::
inline void document_stream::start() noexcept {
if (error) { return; }
error = parser.ensure_capacity(batch_size);
error = parser->ensure_capacity(batch_size);
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
error = run_stage1(parser, batch_start);
error = run_stage1(*parser, batch_start);
if (error) { return; }
#ifdef SIMDJSON_THREADS_ENABLED
@ -149,8 +157,8 @@ inline void document_stream::next() noexcept {
if (error) { return; }
// Load the next document from the batch
doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index];
error = parser.implementation->stage2_next(parser.doc);
doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index];
error = parser->implementation->stage2_next(parser->doc);
// If that was the last document in the batch, load another batch (if available)
while (error == EMPTY) {
batch_start = next_batch_start();
@ -159,17 +167,17 @@ inline void document_stream::next() noexcept {
#ifdef SIMDJSON_THREADS_ENABLED
load_from_stage1_thread();
#else
error = run_stage1(parser, batch_start);
error = run_stage1(*parser, batch_start);
#endif
if (error) { continue; } // If the error was EMPTY, we may want to load another batch.
// Run stage 2 on the first document in the batch
doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index];
error = parser.implementation->stage2_next(parser.doc);
doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index];
error = parser->implementation->stage2_next(parser->doc);
}
}
inline size_t document_stream::next_batch_start() const noexcept {
return batch_start + parser.implementation->structural_indexes[parser.implementation->n_structural_indexes];
return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes];
}
inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept {
@ -188,7 +196,7 @@ inline void document_stream::load_from_stage1_thread() noexcept {
worker->finish();
// Swap to the parser that was loaded up in the thread. Make sure the parser has
// enough memory to swap to, as well.
std::swap(parser, stage1_thread_parser);
std::swap(*parser, stage1_thread_parser);
error = stage1_thread_error;
if (error) { return; }

4
include/simdjson/inline/error.h
View File

@ -53,9 +53,7 @@ really_inline void simdjson_result_base<T>::tie(T &value, error_code &error) &&
template<typename T>
WARN_UNUSED really_inline error_code simdjson_result_base<T>::get(T &value) && noexcept {
error_code error;
std::forward<simdjson_result_base<T>>(*this).tie(value, error);
return error;
return std::forward<simdjson_result_base<T>>(*this).get(value);
}
template<typename T>

42
tests/basictests.cpp
View File

@ -385,7 +385,9 @@ namespace document_tests {
namespace document_stream_tests {
static simdjson::dom::document_stream parse_many_stream_return(simdjson::dom::parser &parser, simdjson::padded_string &str) {
return parser.parse_many(str);
simdjson::dom::document_stream stream;
UNUSED auto error = parser.parse_many(str).get(stream);
return stream;
}
// this is a compilation test
UNUSED static void parse_many_stream_assign() {
@ -1032,11 +1034,11 @@ namespace dom_api_tests {
if (doc["obj"]["a"].get<uint64_t>().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; }
object obj;
error = doc.get(obj); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0
error = doc.get(obj);
if (error) { cerr << "Error: " << error << endl; return false; }
if (obj["obj"]["a"].get<uint64_t>().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; }
error = obj["obj"].get(obj); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0
error = obj["obj"].get(obj);
if (obj["a"].get<uint64_t>().first != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << obj["a"].first << endl; return false; }
if (obj["b"].get<uint64_t>().first != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << obj["b"].first << endl; return false; }
if (obj["c/d"].get<uint64_t>().first != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << obj["c"].first << endl; return false; }
@ -1065,19 +1067,17 @@ namespace dom_api_tests {
// Print users with a default profile.
set<string_view> default_users;
dom::parser parser;
auto [tweets, error] = parser.load(TWITTER_JSON)["statuses"].get<dom::array>();
dom::array tweets;
auto error = parser.load(TWITTER_JSON)["statuses"].get(tweets);
if (error) { cerr << "Error: " << error << endl; return false; }
for (auto tweet : tweets) {
object user;
error = tweet["user"].get(user); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0;
if (error) { cerr << "Error: " << error << endl; return false; }
if ((error = tweet["user"].get(user))) { cerr << "Error: " << error << endl; return false; }
bool default_profile;
error = user["default_profile"].get(default_profile); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0;
if (error) { cerr << "Error: " << error << endl; return false; }
if ((error = user["default_profile"].get(default_profile))) { cerr << "Error: " << error << endl; return false; }
if (default_profile) {
std::string_view screen_name;
error = user["screen_name"].get(screen_name); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0;
if (error) { cerr << "Error: " << error << endl; return false; }
if ((error = user["screen_name"].get(screen_name))) { cerr << "Error: " << error << endl; return false; }
default_users.insert(screen_name);
}
}
@ -1090,21 +1090,19 @@ namespace dom_api_tests {
// Print image names and sizes
set<pair<uint64_t, uint64_t>> image_sizes;
dom::parser parser;
auto [tweets, error] = parser.load(TWITTER_JSON)["statuses"].get<dom::array>();
dom::array tweets;
auto error = parser.load(TWITTER_JSON)["statuses"].get(tweets);
if (error) { cerr << "Error: " << error << endl; return false; }
for (auto tweet : tweets) {
auto [media, not_found] = tweet["entities"]["media"].get<dom::array>();
if (!not_found) {
dom::array media;
if (not (error = tweet["entities"]["media"].get(media))) {
for (auto image : media) {
object sizes;
error = image["sizes"].get(sizes); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0;
if (error) { cerr << "Error: " << error << endl; return false; }
if ((error = image["sizes"].get(sizes))) { cerr << "Error: " << error << endl; return false; }
for (auto size : sizes) {
uint64_t width, height;
error = size.value["w"].get(width); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0;
if (error) { cerr << "Error: " << error << endl; return false; }
error = size.value["h"].get(height); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0;
if (error) { cerr << "Error: " << error << endl; return false; }
if ((error = size.value["w"].get(width))) { cerr << "Error: " << error << endl; return false; }
if ((error = size.value["h"].get(height))) { cerr << "Error: " << error << endl; return false; }
image_sizes.insert(make_pair(width, height));
}
}
@ -1343,7 +1341,9 @@ namespace type_tests {
// Grab the element out and check success
dom::element element = result.first;
RUN_TEST( tester.test_get(element, expected ) );
RUN_TEST( tester.test_get_t(element, expected) );
RUN_TEST( tester.test_get_t(result, expected) );
RUN_TEST( tester.test_get(element, expected) );
RUN_TEST( tester.test_get(result, expected) );
// RUN_TEST( tester.test_named_get(element, expected) );
// RUN_TEST( tester.test_named_get(result, expected) );
@ -1366,6 +1366,8 @@ namespace type_tests {
// Grab the element out and check success
dom::element element = result.first;
RUN_TEST( tester.test_get_t(element) );
RUN_TEST( tester.test_get_t(result) );
RUN_TEST( tester.test_get(element) );
RUN_TEST( tester.test_get(result) );
RUN_TEST( tester.test_named_get(element) );

71
tests/cast_tester.h
View File

@ -26,6 +26,11 @@ public:
bool test_get_error(element element, error_code expected_error);
bool test_get_error(simdjson_result<element> element, error_code expected_error);
bool test_get_t(element element, T expected = {});
bool test_get_t(simdjson_result<element> element, T expected = {});
bool test_get_t_error(element element, error_code expected_error);
bool test_get_t_error(simdjson_result<element> element, error_code expected_error);
#if SIMDJSON_EXCEPTIONS
bool test_implicit_cast(element element, T expected = {});
bool test_implicit_cast(simdjson_result<element> element, T expected = {});
@ -57,68 +62,82 @@ private:
template<typename T>
bool cast_tester<T>::test_get(element element, T expected) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(element.get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_get(simdjson_result<element> element, T expected) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(element.get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_get_error(element element, error_code expected_error) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(element.get(actual), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_get_error(simdjson_result<element> element, error_code expected_error) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(element.get(actual), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_get_t(element element, T expected) {
auto actual = element.get<T>();
ASSERT_SUCCESS(actual.error());
return assert_equal(actual.first, expected);
}
template<typename T>
bool cast_tester<T>::test_get_t(simdjson_result<element> element, T expected) {
auto actual = element.get<T>();
ASSERT_SUCCESS(actual.error());
return assert_equal(actual.first, expected);
}
template<typename T>
bool cast_tester<T>::test_get_t_error(element element, error_code expected_error) {
ASSERT_EQUAL(element.get<T>().error(), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_get_t_error(simdjson_result<element> element, error_code expected_error) {
ASSERT_EQUAL(element.get<T>().error(), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_named_get(element element, T expected) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(named_get(element).get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_named_get(simdjson_result<element> element, T expected) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(named_get(element).get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_named_get_error(element element, error_code expected_error) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(named_get(element).get(actual), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_named_get_error(simdjson_result<element> element, error_code expected_error) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(named_get(element).get(actual), expected_error);
return true;
}
@ -188,8 +207,7 @@ bool cast_tester<T>::test_is(element element, bool expected) {
template<typename T>
bool cast_tester<T>::test_is(simdjson_result<element> element, bool expected) {
bool actual;
auto error = element.is<T>().get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(element.is<T>().get(actual));
ASSERT_EQUAL(actual, expected);
return true;
}
@ -197,8 +215,7 @@ bool cast_tester<T>::test_is(simdjson_result<element> element, bool expected) {
template<typename T>
bool cast_tester<T>::test_is_error(simdjson_result<element> element, error_code expected_error) {
UNUSED bool actual;
auto error = element.is<T>().get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(element.is<T>().get(actual), expected_error);
return true;
}
@ -211,8 +228,7 @@ bool cast_tester<T>::test_named_is(element element, bool expected) {
template<typename T>
bool cast_tester<T>::test_named_is(simdjson_result<element> element, bool expected) {
bool actual;
auto error = named_is(element).get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(named_is(element).get(actual));
ASSERT_EQUAL(actual, expected);
return true;
}
@ -220,8 +236,7 @@ bool cast_tester<T>::test_named_is(simdjson_result<element> element, bool expect
template<typename T>
bool cast_tester<T>::test_named_is_error(simdjson_result<element> element, error_code expected_error) {
bool actual;
auto error = named_is(element).get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(named_is(element).get(actual), expected_error);
return true;
}

17
tests/errortests.cpp
View File

@ -21,6 +21,7 @@ const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json";
#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; }
#define ASSERT_ERROR(ACTUAL, EXPECTED) if ((ACTUAL) != (EXPECTED)) { cerr << "FAIL: Unexpected error \"" << (ACTUAL) << "\" (expected \"" << (EXPECTED) << "\")" << endl; return false; }
#define ASSERT_SUCCESS(CODE) do { simdjson::error_code error = CODE; if (error) { cerr << "FAIL: Unexpected error " << error << endl; return false; } } while (0);
#define TEST_FAIL(MESSAGE) { cerr << "FAIL: " << (MESSAGE) << endl; return false; }
#define TEST_SUCCEED() { return true; }
namespace parser_load {
@ -35,7 +36,9 @@ namespace parser_load {
bool parser_load_many_capacity() {
TEST_START();
dom::parser parser(1); // 1 byte max capacity
for (auto doc : parser.load_many(TWITTER_JSON)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.load_many(TWITTER_JSON).get(docs));
for (auto doc : docs) {
ASSERT_ERROR(doc.error(), CAPACITY);
TEST_SUCCEED();
}
@ -47,7 +50,9 @@ namespace parser_load {
const padded_string DOC = "1 2 [} 3"_padded;
size_t count = 0;
dom::parser parser;
for (auto doc : parser.parse_many(DOC)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
auto [val, error] = doc.get<uint64_t>();
if (count == 3) {
@ -66,7 +71,9 @@ namespace parser_load {
const padded_string DOC = "["_padded;
size_t count = 0;
dom::parser parser;
for (auto doc : parser.parse_many(DOC)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
ASSERT_ERROR(doc.error(), TAPE_ERROR);
}
@ -79,7 +86,9 @@ namespace parser_load {
const padded_string DOC = "1 2 ["_padded;
size_t count = 0;
dom::parser parser;
for (auto doc : parser.parse_many(DOC)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
auto [val, error] = doc.get<uint64_t>();
if (count == 3) {

9
tests/parse_many_test.cpp
View File

@ -69,13 +69,16 @@ bool validate(const char *dirname) {
snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name);
/* The actual test*/
auto [json, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string json;
auto error = simdjson::padded_string::load(fullpath).get(json);
if (!error) {
simdjson::dom::parser parser;
++how_many;
for (auto result : parser.parse_many(json)) {
error = result.error();
simdjson::dom::document_stream docs;
error = parser.parse_many(json).get(docs);
for (auto doc : docs) {
error = doc.error();
}
}
printf("%s\n", error ? "ok" : "invalid");

4
tests/test_macros.h
View File

@ -26,9 +26,9 @@ template<>
bool equals_expected<const char *>(const char *actual, const char *expected) {
return !strcmp(actual, expected);
}
#define ASSERT_EQUAL(ACTUAL, EXPECTED) if (!equals_expected(ACTUAL, EXPECTED)) { std::cerr << "Expected " << #ACTUAL << " to be " << (EXPECTED) << ", got " << (ACTUAL) << " instead!" << std::endl; return false; }
#define ASSERT_EQUAL(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (!equals_expected(_actual, _expected)) { std::cerr << "Expected " << #ACTUAL << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; return false; } } while(0);
#define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; }
#define RUN_TEST(RESULT) if (!RESULT) { return false; }
#define ASSERT_SUCCESS(ERROR) if (ERROR) { std::cerr << (ERROR) << std::endl; return false; }
#define ASSERT_SUCCESS(ERROR) do { auto _error = (ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0);
#endif // TEST_MACROS_H