From c4a0fe1606610048aeefb5941af806f4b0d284cd Mon Sep 17 00:00:00 2001 From: John Keiser Date: Thu, 4 Jun 2020 17:40:15 -0700 Subject: [PATCH] Add tests for parse_many() errors --- tests/basictests.cpp | 57 +++++++++++++++++++++++++++----------- tests/errortests.cpp | 66 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 19 deletions(-) diff --git a/tests/basictests.cpp b/tests/basictests.cpp index cedbda4c..b3312e49 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -515,9 +515,9 @@ namespace parse_api_tests { using namespace simdjson; using namespace simdjson::dom; - const padded_string BASIC_JSON = string("[1,2,3]"); - const padded_string BASIC_NDJSON = string("[1,2,3]\n[4,5,6]"); - // const padded_string EMPTY_NDJSON = string(""); + const padded_string BASIC_JSON = "[1,2,3]"_padded; + const padded_string BASIC_NDJSON = "[1,2,3]\n[4,5,6]"_padded; + const padded_string EMPTY_NDJSON = ""_padded; bool parser_parse() { std::cout << "Running " << __func__ << std::endl; @@ -532,24 +532,48 @@ namespace parse_api_tests { dom::parser parser; int count = 0; for (auto [doc, error] : parser.parse_many(BASIC_NDJSON)) { - if (error) { cerr << error << endl; return false; } + if (error) { cerr << "Error in parse_many: " << endl; return false; } if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } count++; } if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } return true; } - // bool parser_parse_many_empty() { - // std::cout << "Running " << __func__ << std::endl; - // dom::parser parser; - // int count = 0; - // for (auto [doc, error] : parser.parse_many(EMPTY_NDJSON)) { - // if (error) { cerr << error << endl; return false; } - // count++; - // } - // if (count != 0) { cerr << "parse_many returned " << count << " documents, expected 0" << endl; return false; } - // return true; - // } + bool parser_parse_many_empty() { + std::cout << "Running " << __func__ << std::endl; + dom::parser parser; + int count = 0; + for (auto doc : parser.parse_many(EMPTY_NDJSON)) { + if (doc.error()) { cerr << "Error in parse_many: " << doc.error() << endl; return false; } + count++; + } + if (count != 0) { cerr << "parse_many returned " << count << " documents, expected 0" << endl; return false; } + return true; + } + + bool parser_parse_many_empty_batches() { + std::cout << "Running " << __func__ << std::endl; + dom::parser parser; + uint64_t count = 0; + constexpr const int BATCH_SIZE = 128; + uint8_t empty_batches_ndjson[BATCH_SIZE*16+SIMDJSON_PADDING]; + memset(&empty_batches_ndjson[0], ' ', BATCH_SIZE*16+SIMDJSON_PADDING); + memcpy(&empty_batches_ndjson[BATCH_SIZE*3+2], "1", 1); + memcpy(&empty_batches_ndjson[BATCH_SIZE*10+4], "2", 1); + memcpy(&empty_batches_ndjson[BATCH_SIZE*11+6], "3", 1); + for (int i=0; i<16; i++) { + printf("| %.*s |", BATCH_SIZE, &empty_batches_ndjson[BATCH_SIZE*i]); + } + for (auto [doc, error] : parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16)) { + if (error) { cerr << "Error in parse_many: " << error << endl; return false; } + count++; + auto [val, val_error] = doc.get(); + if (val_error) { cerr << "Document is not an unsigned int: " << val_error << endl; return false; } + if (val != count) { cerr << "Expected document #" << count << " to equal " << count << ", but got " << val << " instead!" << endl; return false; } + } + if (count != 3) { cerr << "parse_many returned " << count << " documents, expected 0" << endl; return false; } + return true; + } bool parser_load() { std::cout << "Running " << __func__ << " on " << TWITTER_JSON << std::endl; @@ -633,7 +657,8 @@ namespace parse_api_tests { bool run() { return parser_parse() && parser_parse_many() && -// parser_parse_many_empty() && + parser_parse_many_empty() && + parser_parse_many_empty_batches() && parser_load() && parser_load_many() && #if SIMDJSON_EXCEPTIONS diff --git a/tests/errortests.cpp b/tests/errortests.cpp index 76257626..27f9bfc8 100644 --- a/tests/errortests.cpp +++ b/tests/errortests.cpp @@ -42,6 +42,57 @@ namespace parser_load { TEST_FAIL("No documents returned"); } + bool parser_parse_many_documents_error_in_the_middle() { + TEST_START(); + const padded_string DOC = "1 2 [} 3"_padded; + size_t count = 0; + dom::parser parser; + for (auto doc : parser.parse_many(DOC)) { + count++; + auto [val, error] = doc.get(); + if (count == 3) { + ASSERT_ERROR(error, TAPE_ERROR); + } else { + if (error) { TEST_FAIL(error); } + if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; } + } + } + if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; } + TEST_SUCCEED(); + } + + bool parser_parse_many_documents_partial() { + TEST_START(); + const padded_string DOC = "["_padded; + size_t count = 0; + dom::parser parser; + for (auto doc : parser.parse_many(DOC)) { + count++; + ASSERT_ERROR(doc.error(), TAPE_ERROR); + } + if (count != 1) { cerr << "FAIL: expected no documents and 1 error, got " << count << " total things" << endl; return false; } + TEST_SUCCEED(); + } + + bool parser_parse_many_documents_partial_at_the_end() { + TEST_START(); + const padded_string DOC = "1 2 ["_padded; + size_t count = 0; + dom::parser parser; + for (auto doc : parser.parse_many(DOC)) { + count++; + auto [val, error] = doc.get(); + if (count == 3) { + ASSERT_ERROR(error, TAPE_ERROR); + } else { + if (error) { TEST_FAIL(error); } + if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; } + } + } + if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; } + TEST_SUCCEED(); + } + bool parser_load_nonexistent() { TEST_START(); dom::parser parser; @@ -83,9 +134,18 @@ namespace parser_load { TEST_FAIL("No documents returned"); } bool run() { - return parser_load_capacity() && parser_load_many_capacity() - && parser_load_nonexistent() && parser_load_many_nonexistent() && padded_string_load_nonexistent() - && parser_load_chain() && parser_load_many_chain(); + return true + && parser_load_capacity() + && parser_load_many_capacity() + && parser_load_nonexistent() + && parser_load_many_nonexistent() + && padded_string_load_nonexistent() + && parser_load_chain() + && parser_load_many_chain() + && parser_parse_many_documents_error_in_the_middle() + && parser_parse_many_documents_partial() + && parser_parse_many_documents_partial_at_the_end() + ; } }