New API traversal tests.

This commit is contained in:
Daniel Lemire 2020-06-12 17:42:57 -04:00
parent 4dfbf98e4e
commit d1a54249e7
1 changed files with 147 additions and 14 deletions

View File

@ -48,7 +48,6 @@ void print_stat(const stat_t &s) {
s.true_count, s.false_count); s.true_count, s.false_count);
} }
really_inline void simdjson_process_atom(stat_t &s, really_inline void simdjson_process_atom(stat_t &s,
simdjson::dom::element element) { simdjson::dom::element element) {
if (element.is<double>()) { if (element.is<double>()) {
@ -56,7 +55,7 @@ really_inline void simdjson_process_atom(stat_t &s,
} else if (element.is<bool>()) { } else if (element.is<bool>()) {
simdjson::error_code err; simdjson::error_code err;
bool v; bool v;
element.get<bool>().tie(v,err); element.get<bool>().tie(v, err);
if (v) { if (v) {
s.true_count++; s.true_count++;
} else { } else {
@ -71,9 +70,13 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
if (element.is<simdjson::dom::array>()) { if (element.is<simdjson::dom::array>()) {
s.array_count++; s.array_count++;
auto [array, array_error] = element.get<simdjson::dom::array>(); auto [array, array_error] = element.get<simdjson::dom::array>();
if (array_error) { std::cerr << array_error << std::endl; abort(); } if (array_error) {
std::cerr << array_error << std::endl;
abort();
}
for (auto child : array) { for (auto child : array) {
if (child.is<simdjson::dom::array>() || child.is<simdjson::dom::object>()) { if (child.is<simdjson::dom::array>() ||
child.is<simdjson::dom::object>()) {
simdjson_recurse(s, child); simdjson_recurse(s, child);
} else { } else {
simdjson_process_atom(s, child); simdjson_process_atom(s, child);
@ -82,9 +85,13 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
} else if (element.is<simdjson::dom::object>()) { } else if (element.is<simdjson::dom::object>()) {
s.object_count++; s.object_count++;
auto [object, object_error] = element.get<simdjson::dom::object>(); auto [object, object_error] = element.get<simdjson::dom::object>();
if (object_error) { std::cerr << object_error << std::endl; abort(); } if (object_error) {
std::cerr << object_error << std::endl;
abort();
}
for (auto field : object) { for (auto field : object) {
if (field.value.is<simdjson::dom::array>() || field.value.is<simdjson::dom::object>()) { if (field.value.is<simdjson::dom::array>() ||
field.value.is<simdjson::dom::object>()) {
simdjson_recurse(s, field.value); simdjson_recurse(s, field.value);
} else { } else {
simdjson_process_atom(s, field.value); simdjson_process_atom(s, field.value);
@ -95,8 +102,7 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
} }
} }
never_inline stat_t never_inline stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t s{}; stat_t s{};
simdjson::dom::parser parser; simdjson::dom::parser parser;
auto [doc, error] = parser.parse(p); auto [doc, error] = parser.parse(p);
@ -109,6 +115,114 @@ simdjson_compute_stats(const simdjson::padded_string &p) {
return s; return s;
} }
///
struct Stat {
size_t objectCount;
size_t arrayCount;
size_t numberCount;
size_t stringCount;
size_t trueCount;
size_t falseCount;
size_t nullCount;
size_t memberCount; // Number of members in all objects
size_t elementCount; // Number of elements in all arrays
size_t stringLength; // Number of code units in all strings
};
static void GenStatPlus(Stat &stat, const dom::element &v) {
switch (v.type()) {
case dom::element_type::ARRAY:
for (dom::element child : dom::array(v)) {
GenStatPlus(stat, child);
stat.elementCount++;
}
stat.arrayCount++;
break;
case dom::element_type::OBJECT:
for (dom::key_value_pair kv : dom::object(v)) {
GenStatPlus(stat, dom::element(kv.value));
stat.memberCount++;
stat.stringCount++;
}
stat.objectCount++;
break;
case dom::element_type::INT64:
case dom::element_type::UINT64:
case dom::element_type::DOUBLE:
stat.numberCount++;
break;
case dom::element_type::STRING: {
stat.stringCount++;
std::string_view sv = v.get<std::string_view>();
stat.stringLength += sv.size();
} break;
case dom::element_type::BOOL:
if (v.get<bool>()) {
stat.trueCount++;
} else {
stat.falseCount++;
}
break;
case dom::element_type::NULL_VALUE:
++stat.nullCount;
break;
}
}
static void RapidGenStat(Stat &stat, const rapidjson::Value &v) {
switch (v.GetType()) {
case kNullType:
stat.nullCount++;
break;
case kFalseType:
stat.falseCount++;
break;
case kTrueType:
stat.trueCount++;
break;
case kObjectType:
for (Value::ConstMemberIterator m = v.MemberBegin(); m != v.MemberEnd();
++m) {
stat.stringLength += m->name.GetStringLength();
RapidGenStat(stat, m->value);
}
stat.objectCount++;
stat.memberCount += (v.MemberEnd() - v.MemberBegin());
stat.stringCount += (v.MemberEnd() - v.MemberBegin()); // Key
break;
case kArrayType:
for (Value::ConstValueIterator i = v.Begin(); i != v.End(); ++i)
RapidGenStat(stat, *i);
stat.arrayCount++;
stat.elementCount += v.Size();
break;
case kStringType:
stat.stringCount++;
stat.stringLength += v.GetStringLength();
break;
case kNumberType:
stat.numberCount++;
break;
}
}
never_inline Stat rapidjson_compute_stats_ref(const rapidjson::Value &doc) {
Stat s{};
RapidGenStat(s, doc);
return s;
}
never_inline Stat
simdjson_compute_stats_refplus(const simdjson::dom::element &doc) {
Stat s{};
GenStatPlus(s, doc);
return s;
}
// see // see
// https://github.com/miloyip/nativejson-benchmark/blob/master/src/tests/sajsontest.cpp // https://github.com/miloyip/nativejson-benchmark/blob/master/src/tests/sajsontest.cpp
void sajson_traverse(stat_t &stats, const sajson::value &node) { void sajson_traverse(stat_t &stats, const sajson::value &node) {
@ -152,8 +266,7 @@ void sajson_traverse(stat_t &stats, const sajson::value &node) {
} }
} }
never_inline stat_t never_inline stat_t sasjon_compute_stats(const simdjson::padded_string &p) {
sasjon_compute_stats(const simdjson::padded_string &p) {
stat_t answer{}; stat_t answer{};
char *buffer = (char *)malloc(p.size()); char *buffer = (char *)malloc(p.size());
if (buffer == nullptr) { if (buffer == nullptr) {
@ -214,8 +327,7 @@ void rapid_traverse(stat_t &stats, const rapidjson::Value &v) {
} }
} }
never_inline stat_t never_inline stat_t rapid_compute_stats(const simdjson::padded_string &p) {
rapid_compute_stats(const simdjson::padded_string &p) {
stat_t answer{}; stat_t answer{};
char *buffer = (char *)malloc(p.size() + 1); char *buffer = (char *)malloc(p.size() + 1);
if (buffer == nullptr) { if (buffer == nullptr) {
@ -344,8 +456,29 @@ int main(int argc, char *argv[]) {
repeat, volume, !just_data); repeat, volume, !just_data);
BEST_TIME("RapidJSON ", rapid_compute_stats(p).valid, true, , BEST_TIME("RapidJSON ", rapid_compute_stats(p).valid, true, ,
repeat, volume, !just_data); repeat, volume, !just_data);
BEST_TIME("RapidJSON (precise) ", rapid_accurate_compute_stats(p).valid, true, , BEST_TIME("RapidJSON (precise) ", rapid_accurate_compute_stats(p).valid, true,
repeat, volume, !just_data); , repeat, volume, !just_data);
BEST_TIME("sasjon ", sasjon_compute_stats(p).valid, true, , BEST_TIME("sasjon ", sasjon_compute_stats(p).valid, true, ,
repeat, volume, !just_data); repeat, volume, !just_data);
if (!just_data) {
printf("API traversal tests\n");
simdjson::dom::parser parser;
auto [doc, err] = parser.parse(p);
if (err) {
std::cerr << err << std::endl;
}
size_t refval = simdjson_compute_stats_refplus(doc).objectCount;
BEST_TIME("simdjson ",
simdjson_compute_stats_refplus(doc).objectCount, refval, , repeat,
volume, !just_data);
char *buffer = (char *)malloc(p.size() + 1);
memcpy(buffer, p.data(), p.size());
buffer[p.size()] = '\0';
rapidjson::Document d;
d.ParseInsitu<kParseValidateEncodingFlag>(buffer);
BEST_TIME("rapid ", rapidjson_compute_stats_ref(d).objectCount,
refval, , repeat, volume, !just_data);
free(buffer);
}
} }