jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Added a version of RapidJSON with static alloc.

This commit is contained in:
Daniel Lemire 2018-12-12 10:19:32 -05:00
parent 751dce98f5
commit 15161669ec
1 changed files with 99 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
benchmark/parsingcompetition.cpp
View File

@ -1,5 +1,5 @@
#include <unistd.h>
#include "simdjson/jsonparser.h" #include "simdjson/jsonparser.h"
#include <unistd.h>
#include "benchmark.h" #include "benchmark.h"
@ -10,28 +10,27 @@
#include "rapidjson/stringbuffer.h" #include "rapidjson/stringbuffer.h"
#include "rapidjson/writer.h" #include "rapidjson/writer.h"
#include "json11.cpp"
#include "sajson.h"
#include "fastjson.cpp" #include "fastjson.cpp"
#include "fastjson_dom.cpp" #include "fastjson_dom.cpp"
#include "gason.cpp" #include "gason.cpp"
extern "C" #include "json11.cpp"
{ #include "sajson.h"
#include "ultrajsondec.c" extern "C" {
#include "ujdecode.h" #include "ujdecode.h"
#include "ultrajsondec.c"
} }
using namespace rapidjson; using namespace rapidjson;
using namespace std; using namespace std;
// fastjson has a tricky interface // fastjson has a tricky interface
void on_json_error( void *, const fastjson::ErrorContext& ec) { void on_json_error(void *, const fastjson::ErrorContext &ec) {
//std::cerr<<"ERROR: "<<ec.mesg<<std::endl; // std::cerr<<"ERROR: "<<ec.mesg<<std::endl;
} }
bool fastjson_parse(const char *input) { bool fastjson_parse(const char *input) {
fastjson::Token token; fastjson::Token token;
fastjson::dom::Chunk chunk; fastjson::dom::Chunk chunk;
return fastjson::dom::parse_string(input, &token, &chunk, 0, &on_json_error, NULL); return fastjson::dom::parse_string(input, &token, &chunk, 0, &on_json_error,
NULL);
} }
// end of fastjson stuff // end of fastjson stuff
@ -39,9 +38,8 @@ int main(int argc, char *argv[]) {
bool verbose = false; bool verbose = false;
bool all = false; bool all = false;
int c; int c;
while ((c = getopt (argc, argv, "va")) != -1) while ((c = getopt(argc, argv, "va")) != -1)
switch (c) switch (c) {
{
case 'v': case 'v':
verbose = true; verbose = true;
break; break;
@ -49,22 +47,23 @@ int main(int argc, char *argv[]) {
all = true; all = true;
break; break;
default: default:
abort (); abort();
} }
if (optind >= argc) { if (optind >= argc) {
cerr << "Usage: " << argv[0] << " <jsonfile>\n"; cerr << "Usage: " << argv[0] << " <jsonfile>\n";
cerr << "Or " << argv[0] << " -v <jsonfile>\n"; cerr << "Or " << argv[0] << " -v <jsonfile>\n";
cerr << "To enable parsers that are not standard compliant, use the -a flag\n"; cerr << "To enable parsers that are not standard compliant, use the -a "
"flag\n";
exit(1); exit(1);
} }
const char * filename = argv[optind]; const char *filename = argv[optind];
if(optind + 1 < argc) { if (optind + 1 < argc) {
cerr << "warning: ignoring everything after " << argv[optind + 1] << endl; cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
} }
std::string_view p; std::string_view p;
try { try {
p = get_corpus(filename); p = get_corpus(filename);
} catch (const std::exception& e) { // caught by reference to base } catch (const std::exception &e) { // caught by reference to base
std::cout << "Could not load the file " << filename << std::endl; std::cout << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE; return EXIT_FAILURE;
} }
@ -88,42 +87,95 @@ int main(int argc, char *argv[]) {
} }
int repeat = 10; int repeat = 10;
int volume = p.size(); int volume = p.size();
BEST_TIME("simdjson (dynamic mem) ", build_parsed_json(p).isValid(), true, , repeat, volume, true); BEST_TIME("simdjson (dynamic mem) ", build_parsed_json(p).isValid(), true, ,
repeat, volume, true);
BEST_TIME("simdjson (static alloc) ", json_parse(p, pj), true, , repeat, volume, true); BEST_TIME("simdjson (static alloc) ", json_parse(p, pj), true, , repeat,
volume, true);
rapidjson::Document d; rapidjson::Document d;
char *buffer = (char *)malloc(p.size() + 1); char *buffer = (char *)malloc(p.size() + 1);
memcpy(buffer, p.data(), p.size()); memcpy(buffer, p.data(), p.size());
buffer[p.size()] = '\0'; buffer[p.size()] = '\0';
//
// Todo: It is possible to preallocate a block of memory with RapidJSON using a MemoryAllocator. BEST_TIME(
BEST_TIME("RapidJSON", "RapidJSON",
d.Parse<kParseValidateEncodingFlag>((const char *)buffer).HasParseError(), d.Parse<kParseValidateEncodingFlag>((const char *)buffer).HasParseError(),
false, memcpy(buffer, p.data(), p.size()), repeat, volume, true); false, memcpy(buffer, p.data(), p.size()), repeat, volume, true);
BEST_TIME("RapidJSON (insitu)", d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError(), false, BEST_TIME("RapidJSON (insitu)",
memcpy(buffer, p.data(), p.size()), repeat, volume, true); d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError(),
false, memcpy(buffer, p.data(), p.size()), repeat, volume, true);
typedef rapidjson::GenericDocument<UTF8<>, rapidjson::MemoryPoolAllocator<>,
rapidjson::MemoryPoolAllocator<>>
RapidDocumentType;
size_t rapidvaallocsize = p.size() * 128; // allocate plenty of memory
size_t rapidallocsize = p.size() * 4096; // allocate plenty of memory
char *rapidvalueBuffer = (char *)malloc(rapidvaallocsize);
char *rapidparseBuffer = (char *)malloc(rapidallocsize);
if ((rapidvalueBuffer != NULL) && (rapidvalueBuffer != NULL)) {
rapidjson::MemoryPoolAllocator<> valueAllocator(rapidvalueBuffer,
rapidvaallocsize);
rapidjson::MemoryPoolAllocator<> parseAllocator(rapidparseBuffer,
rapidallocsize);
RapidDocumentType preallocedd(&valueAllocator, rapidvaallocsize,
&parseAllocator);
BEST_TIME("sajson (dynamic mem, insitu)", sajson::parse(sajson::dynamic_allocation(), sajson::mutable_string_view(p.size(), buffer)).is_valid(), true, memcpy(buffer, p.data(), p.size()), repeat, volume, true); BEST_TIME(
"RapidJSON (static alloc)",
preallocedd.Parse<kParseValidateEncodingFlag>((const char *)buffer)
.HasParseError(),
false, memcpy(buffer, p.data(), p.size()), repeat, volume, true);
BEST_TIME("RapidJSON (static alloc, insitu)",
preallocedd.ParseInsitu<kParseValidateEncodingFlag>(buffer)
.HasParseError(),
false, memcpy(buffer, p.data(), p.size()), repeat, volume, true);
assert(valueAllocator.Size() <= rapidvaallocsize);
assert(parseAllocator.Size() <= rapidallocsize);
}
free(rapidvalueBuffer);
free(rapidparseBuffer);
BEST_TIME("sajson (dynamic mem, insitu)",
sajson::parse(sajson::dynamic_allocation(),
sajson::mutable_string_view(p.size(), buffer))
.is_valid(),
true, memcpy(buffer, p.data(), p.size()), repeat, volume, true);
size_t astbuffersize = p.size(); size_t astbuffersize = p.size();
size_t * ast_buffer = (size_t *) malloc(astbuffersize * sizeof(size_t)); size_t *ast_buffer = (size_t *)malloc(astbuffersize * sizeof(size_t));
BEST_TIME("sajson (static alloc, insitu)", sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize), sajson::mutable_string_view(p.size(), buffer)).is_valid(), true, memcpy(buffer, p.data(), p.size()), repeat, volume, true); BEST_TIME("sajson (static alloc, insitu)",
sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize),
sajson::mutable_string_view(p.size(), buffer))
.is_valid(),
true, memcpy(buffer, p.data(), p.size()), repeat, volume, true);
std::string json11err; std::string json11err;
if(all) BEST_TIME("dropbox (json11) ", (( json11::Json::parse(buffer,json11err).is_null() ) || ( ! json11err.empty() )), false, memcpy(buffer, p.data(), p.size()), repeat, volume, true); if (all)
BEST_TIME("dropbox (json11) ",
((json11::Json::parse(buffer, json11err).is_null()) ||
(!json11err.empty())),
false, memcpy(buffer, p.data(), p.size()), repeat, volume, true);
if(all) BEST_TIME("fastjson ", fastjson_parse(buffer), true, memcpy(buffer, p.data(), p.size()), repeat, volume, true); if (all)
BEST_TIME("fastjson ", fastjson_parse(buffer), true,
memcpy(buffer, p.data(), p.size()), repeat, volume, true);
JsonValue value; JsonValue value;
JsonAllocator allocator; JsonAllocator allocator;
char *endptr; char *endptr;
if(all) BEST_TIME("gason ", jsonParse(buffer, &endptr, &value, allocator), JSON_OK, memcpy(buffer, p.data(), p.size()), repeat, volume, true); if (all)
BEST_TIME("gason ",
jsonParse(buffer, &endptr, &value, allocator), JSON_OK,
memcpy(buffer, p.data(), p.size()), repeat, volume, true);
void *state; void *state;
if(all) BEST_TIME("ultrajson ", (UJDecode(buffer, p.size(), NULL, &state) == NULL), false, memcpy(buffer, p.data(), p.size()), repeat, volume, true); if (all)
BEST_TIME("memcpy ", (memcpy(buffer, p.data(), p.size()) == buffer), true, , repeat, volume, true); BEST_TIME("ultrajson ",
free((void*)p.data()); (UJDecode(buffer, p.size(), NULL, &state) == NULL), false,
memcpy(buffer, p.data(), p.size()), repeat, volume, true);
BEST_TIME("memcpy ",
(memcpy(buffer, p.data(), p.size()) == buffer), true, , repeat,
volume, true);
free((void *)p.data());
free(ast_buffer); free(ast_buffer);
free(buffer); free(buffer);
} }