simdjson/benchmark/minifiercompetition.cpp

182 lines
5.8 KiB
C++
Raw Normal View History

2018-08-21 05:30:00 +08:00
#include <iostream>
#include <unistd.h>
2018-08-21 05:30:00 +08:00
#include "benchmark.h"
2018-11-30 22:37:57 +08:00
#include "simdjson/jsonioutil.h"
#include "simdjson/jsonminifier.h"
#include "simdjson/jsonparser.h"
2018-08-21 05:30:00 +08:00
// #define RAPIDJSON_SSE2 // bad
// #define RAPIDJSON_SSE42 // bad
#include "rapidjson/document.h"
#include "rapidjson/reader.h" // you have to check in the submodule
#include "rapidjson/stringbuffer.h"
#include "rapidjson/writer.h"
2018-09-28 12:00:52 +08:00
#include "sajson.h"
2018-08-21 05:30:00 +08:00
using namespace simdjson;
2018-08-21 05:30:00 +08:00
using namespace rapidjson;
std::string rapid_stringme_insitu(char *json) {
2018-08-21 05:30:00 +08:00
Document d;
d.ParseInsitu(json);
if (d.HasParseError()) {
std::cerr << "problem!" << std::endl;
return ""; // should do something
}
StringBuffer buffer;
Writer<StringBuffer> writer(buffer);
d.Accept(writer);
return buffer.GetString();
}
std::string rapid_stringme(char *json) {
2018-08-21 05:30:00 +08:00
Document d;
d.Parse(json);
if (d.HasParseError()) {
std::cerr << "problem!" << std::endl;
return ""; // should do something
}
StringBuffer buffer;
Writer<StringBuffer> writer(buffer);
d.Accept(writer);
return buffer.GetString();
}
int main(int argc, char *argv[]) {
2018-11-28 03:37:59 +08:00
int c;
bool verbose = false;
bool just_data = false;
while ((c = getopt(argc, argv, "vt")) != -1)
switch (c) {
case 't':
just_data = true;
break;
case 'v':
verbose = true;
break;
default:
abort();
}
2018-11-28 03:37:59 +08:00
if (optind >= argc) {
std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
2018-08-21 05:30:00 +08:00
exit(1);
}
const char *filename = argv[optind];
2019-07-03 03:21:00 +08:00
simdjson::padded_string p;
2018-11-28 03:37:59 +08:00
try {
2019-07-03 03:21:00 +08:00
simdjson::get_corpus(filename).swap(p);
} catch (const std::exception &e) { // caught by reference to base
2018-11-28 03:37:59 +08:00
std::cout << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
2018-08-21 05:30:00 +08:00
}
2018-08-21 05:40:50 +08:00
if (verbose) {
std::cout << "Input has ";
2018-12-01 09:27:16 +08:00
if (p.size() > 1024 * 1024)
std::cout << p.size() / (1024 * 1024) << " MB ";
else if (p.size() > 1024)
std::cout << p.size() / 1024 << " KB ";
2018-08-21 05:40:50 +08:00
else
2018-12-01 09:27:16 +08:00
std::cout << p.size() << " B ";
2018-08-21 05:40:50 +08:00
std::cout << std::endl;
}
2019-07-26 20:37:52 +08:00
char *buffer = simdjson::allocate_padded_buffer(p.size() + 1);
2018-12-01 10:31:05 +08:00
memcpy(buffer, p.data(), p.size());
2018-12-01 09:27:16 +08:00
buffer[p.size()] = '\0';
2018-08-21 05:30:00 +08:00
2018-12-28 06:10:19 +08:00
int repeat = 50;
2018-12-01 09:27:16 +08:00
int volume = p.size();
if (just_data) {
printf(
"name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
2018-12-28 06:10:19 +08:00
}
size_t strlength = rapid_stringme((char *)p.data()).size();
2018-08-21 05:40:50 +08:00
if (verbose)
2018-12-01 09:27:16 +08:00
std::cout << "input length is " << p.size() << " stringified length is "
2018-08-21 05:40:50 +08:00
<< strlength << std::endl;
BEST_TIME_NOCHECK("despacing with RapidJSON",
rapid_stringme((char *)p.data()), , repeat, volume,
!just_data);
BEST_TIME_NOCHECK(
"despacing with RapidJSON Insitu", rapid_stringme_insitu((char *)buffer),
memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
2018-12-01 10:31:05 +08:00
memcpy(buffer, p.data(), p.size());
2018-08-21 05:30:00 +08:00
size_t outlength = simdjson::json_minify((const uint8_t *)buffer, p.size(),
(uint8_t *)buffer);
2018-08-21 05:40:50 +08:00
if (verbose)
std::cout << "json_minify length is " << outlength << std::endl;
2018-08-21 05:30:00 +08:00
uint8_t *cbuffer = (uint8_t *)buffer;
BEST_TIME("json_minify", simdjson::json_minify(cbuffer, p.size(), cbuffer),
outlength, memcpy(buffer, p.data(), p.size()), repeat, volume,
!just_data);
printf("minisize = %zu, original size = %zu (minified down to %.2f percent "
"of original) \n",
outlength, p.size(), outlength * 100.0 / p.size());
2018-08-21 05:30:00 +08:00
/***
* Is it worth it to minify before parsing?
***/
rapidjson::Document d;
BEST_TIME("RapidJSON Insitu orig", d.ParseInsitu(buffer).HasParseError(),
false, memcpy(buffer, p.data(), p.size()), repeat, volume,
!just_data);
char *mini_buffer = simdjson::allocate_padded_buffer(p.size() + 1);
size_t minisize = simdjson::json_minify((const uint8_t *)p.data(), p.size(),
(uint8_t *)mini_buffer);
mini_buffer[minisize] = '\0';
BEST_TIME("RapidJSON Insitu despaced", d.ParseInsitu(buffer).HasParseError(),
false, memcpy(buffer, mini_buffer, p.size()), repeat, volume,
!just_data);
size_t ast_buffer_size = p.size() * 2;
size_t *ast_buffer = (size_t *)malloc(ast_buffer_size * sizeof(size_t));
BEST_TIME(
"sajson orig",
sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
sajson::mutable_string_view(p.size(), buffer))
.is_valid(),
true, memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
BEST_TIME(
"sajson despaced",
sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
sajson::mutable_string_view(minisize, buffer))
.is_valid(),
true, memcpy(buffer, mini_buffer, p.size()), repeat, volume, !just_data);
2018-09-28 12:00:52 +08:00
2019-07-26 20:37:52 +08:00
simdjson::ParsedJson pj;
bool is_alloc_ok = pj.allocate_capacity(p.size(), 1024);
if (!is_alloc_ok) {
fprintf(stderr, "failed to allocate memory\n");
2018-12-01 11:02:32 +08:00
return EXIT_FAILURE;
}
bool automated_reallocation = false;
BEST_TIME("simdjson orig",
simdjson::json_parse((const uint8_t *)buffer, p.size(), pj,
automated_reallocation),
simdjson::SUCCESS, memcpy(buffer, p.data(), p.size()), repeat, volume,
!just_data);
2019-07-26 20:37:52 +08:00
simdjson::ParsedJson pj2;
bool is_alloc_ok2 = pj2.allocate_capacity(p.size(), 1024);
if (!is_alloc_ok2) {
fprintf(stderr, "failed to allocate memory\n");
2018-12-01 11:02:32 +08:00
return EXIT_FAILURE;
}
automated_reallocation = false;
BEST_TIME("simdjson despaced",
simdjson::json_parse((const uint8_t *)buffer, minisize, pj2,
automated_reallocation),
simdjson::SUCCESS, memcpy(buffer, mini_buffer, p.size()), repeat, volume,
!just_data);
2018-08-21 05:30:00 +08:00
free(buffer);
2018-09-28 12:00:52 +08:00
free(ast_buffer);
free(mini_buffer);
2018-08-21 05:30:00 +08:00
}