jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Add error tests, doc_ref_result[] chaining

This commit is contained in:
John Keiser 2020-03-07 13:10:23 -08:00
parent 40c6213d7e
commit ac0899c043
7 changed files with 208 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -69,6 +69,7 @@ objs
/build-plain-*/
/corpus.zip
/distinctuseridcompetition
/errortests
/fuzz/fuzz_dump
/fuzz/fuzz_dump_raw_tape
/fuzz/fuzz_parser
@ -107,6 +108,7 @@ objs
/singleheader/amalgamation_demo
/singleheader/demo
/tests/basictests
/tests/errortests
/tests/jsoncheck
/tests/pointercheck
/tests/integer_tests

18
Makefile
View File

@ -23,7 +23,7 @@ ARCHFLAGS ?= -msse4.2 -mpclmul # lowest supported feature set?
endif
CXXFLAGS = $(ARCHFLAGS) -std=c++17 -pthread -Wall -Wextra -Wshadow -Ibenchmark/linux
CFLAGS = $(ARCHFLAGS) -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS)
CFLAGS = $(ARCHFLAGS) -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS)
# This is a convenience flag
ifdef SANITIZEGOLD
@ -39,16 +39,16 @@ endif
# SANITIZE *implies* DEBUG
ifeq ($(MEMSANITIZE),1)
CXXFLAGS += -g3 -O0 -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined
CFLAGS += -g3 -O0 -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined
CXXFLAGS += -g3 -O0 -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined
CFLAGS += -g3 -O0 -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined
else
ifeq ($(SANITIZE),1)
CXXFLAGS += -g3 -O0 -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined
CFLAGS += -g3 -O0 -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined
else
ifeq ($(DEBUG),1)
CXXFLAGS += -g3 -O0
CFLAGS += -g3 -O0
CXXFLAGS += -g3 -O0
CFLAGS += -g3 -O0
else
# we opt for -O3 for regular builds
CXXFLAGS += -O3
@ -95,7 +95,7 @@ JSON_INCLUDE:=dependencies/json/single_include/nlohmann/json.hpp
EXTRAOBJECTS=ujdecode.o
MAINEXECUTABLES=parse minify json2json jsonstats statisticalmodel jsonpointer get_corpus_benchmark
TESTEXECUTABLES=jsoncheck jsoncheck_noavx integer_tests numberparsingcheck stringparsingcheck pointercheck parse_many_test basictests readme_examples
TESTEXECUTABLES=jsoncheck jsoncheck_noavx integer_tests numberparsingcheck stringparsingcheck pointercheck parse_many_test basictests errortests readme_examples
COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompetition distinctuseridcompetition allparserscheckfile allparsingcompetition
SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing
@ -112,6 +112,9 @@ benchmark:
run_basictests: basictests
./basictests
run_errortests: errortests
./errortests
run_readme_examples: readme_examples
./readme_examples
@ -217,6 +220,9 @@ jsoncheck_noavx:tests/jsoncheck.cpp $(HEADERS) $(LIBFILES)
basictests:tests/basictests.cpp $(HEADERS) $(LIBFILES)
$(CXX) $(CXXFLAGS) -o basictests tests/basictests.cpp -I. $(LIBFILES) $(LIBFLAGS)
errortests:tests/errortests.cpp $(HEADERS) $(LIBFILES)
$(CXX) $(CXXFLAGS) -o errortests tests/errortests.cpp -I. $(LIBFILES) $(LIBFLAGS)
readme_examples: tests/readme_examples.cpp $(HEADERS) $(LIBFILES)
$(CXX) $(CXXFLAGS) -o readme_examples tests/readme_examples.cpp -I. $(LIBFILES) $(LIBFLAGS)

57
include/simdjson/document.h
View File

@ -265,6 +265,33 @@ public:
*/
operator document() noexcept(false);
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
inline element_result<element> operator[](const std::string_view &key) const noexcept;
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
inline element_result<element> operator[](const char *key) const noexcept;
~doc_result() noexcept=default;
private:
@ -324,6 +351,34 @@ public:
*/
operator document&() noexcept(false);
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
inline element_result<element> operator[](const std::string_view &key) const noexcept;
/**
* Get the value associated with the given key.
*
* The key will be matched against **unescaped** JSON:
*
* document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1
* document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
* - UNEXPECTED_TYPE if the document is not an object
*/
inline element_result<element> operator[](const char *key) const noexcept;
~doc_ref_result()=default;
private:
@ -549,6 +604,7 @@ public:
* - UNEXPECTED_TYPE if the document is not an object
*/
inline element_result<element> operator[](const std::string_view &s) const noexcept;
/**
* Get the value associated with the given key.
*
@ -685,6 +741,7 @@ public:
* - NO_SUCH_FIELD if the field does not exist in the object
*/
inline element_result<element> operator[](const std::string_view &s) const noexcept;
/**
* Get the value associated with the given key.
*

25
include/simdjson/inline/document.h
View File

@ -444,11 +444,17 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
//
inline document::doc_ref_result::doc_ref_result(document &_doc, error_code _error) noexcept : doc(_doc), error(_error) { }
inline document::doc_ref_result::operator document&() noexcept(false) {
if (error) {
throw simdjson_error(error);
}
if (error) { throw simdjson_error(error); }
return doc;
}
inline document::element_result<document::element> document::doc_ref_result::operator[](const std::string_view &key) const noexcept {
if (error) { return error; }
return doc[key];
}
inline document::element_result<document::element> document::doc_ref_result::operator[](const char *key) const noexcept {
if (error) { return error; }
return doc[key];
}
//
// document::doc_result inline implementation
@ -457,11 +463,17 @@ inline document::doc_result::doc_result(document &&_doc, error_code _error) noex
inline document::doc_result::doc_result(document &&_doc) noexcept : doc(std::move(_doc)), error(SUCCESS) { }
inline document::doc_result::doc_result(error_code _error) noexcept : doc(), error(_error) { }
inline document::doc_result::operator document() noexcept(false) {
if (error) {
throw simdjson_error(error);
}
if (error) { throw simdjson_error(error); }
return std::move(doc);
}
inline document::element_result<document::element> document::doc_result::operator[](const std::string_view &key) const noexcept {
if (error) { return error; }
return doc[key];
}
inline document::element_result<document::element> document::doc_result::operator[](const char *key) const noexcept {
if (error) { return error; }
return doc[key];
}
//
// document::parser inline implementation
@ -871,7 +883,6 @@ inline document::element_result<int64_t> document::element::as_int64_t() const n
case tape_type::INT64:
return next_tape_value<int64_t>();
default:
std::cout << "Incorrect " << json_index << " = " << char(type()) << std::endl;
return INCORRECT_TYPE;
}
}

2
include/simdjson/inline/document_stream.h
View File

@ -104,7 +104,7 @@ really_inline document::stream::stream(
size_t batch_size,
error_code _error
) noexcept : parser{_parser}, _buf{buf}, _len{len}, _batch_size(batch_size), error{_error} {
error = json_parse();
if (!error) { error = json_parse(); }
}
inline document::stream::~stream() noexcept {

2
tests/CMakeLists.txt
View File

@ -5,6 +5,7 @@ if(MSVC)
endif()
add_cpp_test(basictests)
add_cpp_test(errortests)
## Next bit should not be needed!
#if(CMAKE_INTERPROCEDURAL_OPTIMIZATION)
@ -19,6 +20,7 @@ add_cpp_test(pointercheck)
add_cpp_test(integer_tests)
target_compile_definitions(basictests PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json")
target_compile_definitions(errortests PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json")
## This causes problems
# add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp)

116
tests/errortests.cpp Normal file
View File

@ -0,0 +1,116 @@
#include <cassert>
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <string_view>
#include "simdjson.h"
using namespace simdjson;
using namespace std;
#ifndef JSON_TEST_PATH
#define JSON_TEST_PATH "jsonexamples/twitter.json"
#endif
#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; }
#define ASSERT_ERROR(ACTUAL, EXPECTED) if ((ACTUAL) != (EXPECTED)) { cerr << "FAIL: Unexpected error \"" << (ACTUAL) << "\" (expected \"" << (EXPECTED) << "\")" << endl; return false; }
#define TEST_FAIL(MESSAGE) { cerr << "FAIL: " << (MESSAGE) << endl; return false; }
#define TEST_SUCCEED() { return true; }
namespace parser_load {
const char * NONEXISTENT_FILE = "this_file_does_not_exit.json";
bool parser_load_capacity() {
TEST_START();
document::parser parser(1); // 1 byte max capacity
auto [doc, error] = parser.load(JSON_TEST_PATH);
ASSERT_ERROR(error, CAPACITY);
TEST_SUCCEED();
}
bool parser_load_many_capacity() {
TEST_START();
document::parser parser(1); // 1 byte max capacity
for (auto [doc, error] : parser.load_many(JSON_TEST_PATH)) {
ASSERT_ERROR(error, CAPACITY);
TEST_SUCCEED();
}
TEST_FAIL("No documents returned");
}
bool document_load_nonexistent() {
TEST_START();
auto [doc, error] = document::load(NONEXISTENT_FILE);
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
bool parser_load_nonexistent() {
TEST_START();
document::parser parser;
auto [doc, error] = parser.load(NONEXISTENT_FILE);
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
bool parser_load_many_nonexistent() {
TEST_START();
document::parser parser;
for (auto [doc, error] : parser.load_many(NONEXISTENT_FILE)) {
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
TEST_FAIL("No documents returned");
}
bool padded_string_load_nonexistent() {
TEST_START();
auto [str, error] = padded_string::load(NONEXISTENT_FILE);
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
bool document_load_chain() {
TEST_START();
auto [val, error] = document::load(NONEXISTENT_FILE)["foo"].as_uint64_t();
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
bool parser_load_chain() {
TEST_START();
document::parser parser;
auto [val, error] = parser.load(NONEXISTENT_FILE)["foo"].as_uint64_t();
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
bool parser_load_many_chain() {
TEST_START();
document::parser parser;
for (auto doc_result : parser.load_many(NONEXISTENT_FILE)) {
auto [val, error] = doc_result["foo"].as_uint64_t();
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
TEST_FAIL("No documents returned");
}
bool run() {
return parser_load_capacity() && parser_load_many_capacity()
&& parser_load_nonexistent() && parser_load_many_nonexistent() && document_load_nonexistent() && padded_string_load_nonexistent()
&& document_load_chain() && parser_load_chain() && parser_load_many_chain();
}
}
int main() {
// this is put here deliberately to check that the documentation is correct (README),
// should this fail to compile, you should update the documentation:
if (simdjson::active_implementation->name() == "unsupported") {
printf("unsupported CPU\n");
}
std::cout << "Running error tests." << std::endl;
if (!parser_load::run()) {
return EXIT_FAILURE;
}
std::cout << "Error tests are ok." << std::endl;
return EXIT_SUCCESS;
}