#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "linux-perf-events.h" //#define DEBUG #include "simdjson/common_defs.h" #include "simdjson/jsonparser.h" #include "simdjson/jsonioutil.h" #include "simdjson/parsedjson.h" #include "simdjson/stage1_find_marks.h" #include "simdjson/stage2_flatten.h" #include "simdjson/stage34_unified.h" using namespace std; int main(int argc, char *argv[]) { bool verbose = false; bool dump = false; bool forceoneiteration = false; int c; while ((c = getopt (argc, argv, "1vd")) != -1) switch (c) { case 'v': verbose = true; break; case 'd': dump = true; break; case '1': forceoneiteration = true; break; default: abort (); } if (optind >= argc) { cerr << "Usage: " << argv[0] << " " << endl; exit(1); } const char * filename = argv[optind]; if(optind + 1 < argc) { cerr << "warning: ignoring everything after " << argv[optind + 1] << endl; } if(verbose) cout << "[verbose] loading " << filename << endl; std::string_view p; try { p = get_corpus(filename); } catch (const std::exception& e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; } if(verbose) cout << "[verbose] loaded " << filename << " ("<< p.size() << " bytes)" << endl; ParsedJson pj; bool allocok = pj.allocateCapacity(p.size(), 1024); if(!allocok) { std::cerr << "failed to allocate memory" << std::endl; return EXIT_FAILURE; } if(verbose) cout << "[verbose] allocated memory for parsed JSON " << endl; #if defined(DEBUG) const u32 iterations = 1; #else const u32 iterations = forceoneiteration ? 1 : ( p.size() < 1 * 1000 * 1000? 1000 : 10); #endif vector res; res.resize(iterations); #if !defined(__linux__) #define SQUASH_COUNTERS #endif #ifndef SQUASH_COUNTERS vector evts; evts.push_back(PERF_COUNT_HW_CPU_CYCLES); evts.push_back(PERF_COUNT_HW_INSTRUCTIONS); evts.push_back(PERF_COUNT_HW_BRANCH_MISSES); evts.push_back(PERF_COUNT_HW_CACHE_REFERENCES); evts.push_back(PERF_COUNT_HW_CACHE_MISSES); LinuxEvents unified(evts); vector results; results.resize(evts.size()); unsigned long cy1 = 0, cy2 = 0, cy3 = 0; unsigned long cl1 = 0, cl2 = 0, cl3 = 0; unsigned long mis1 = 0, mis2 = 0, mis3 = 0; unsigned long cref1 = 0, cref2 = 0, cref3 = 0; unsigned long cmis1 = 0, cmis2 = 0, cmis3 = 0; #endif bool isok = true; for (u32 i = 0; i < iterations; i++) { if(verbose) cout << "[verbose] iteration # " << i << endl; auto start = std::chrono::steady_clock::now(); #ifndef SQUASH_COUNTERS unified.start(); #endif isok = find_structural_bits(p.data(), p.size(), pj); #ifndef SQUASH_COUNTERS unified.end(results); cy1 += results[0]; cl1 += results[1]; mis1 += results[2]; cref1 += results[3]; cmis1 += results[4]; if (!isok) { cout << "Failed out during stage 1\n"; break; } unified.start(); #endif isok = isok && flatten_indexes(p.size(), pj); #ifndef SQUASH_COUNTERS unified.end(results); cy2 += results[0]; cl2 += results[1]; mis2 += results[2]; cref2 += results[3]; cmis2 += results[4]; if (!isok) { cout << "Failed out during stage 2\n"; break; } unified.start(); #endif isok = isok && unified_machine(p.data(), p.size(), pj); #ifndef SQUASH_COUNTERS unified.end(results); cy3 += results[0]; cl3 += results[1]; mis3 += results[2]; cref3 += results[3]; cmis3 += results[4]; if (!isok) { cout << "Failed out during stage 34\n"; break; } #endif auto end = std::chrono::steady_clock::now(); std::chrono::duration secs = end - start; res[i] = secs.count(); } #ifndef SQUASH_COUNTERS printf("number of bytes %ld number of structural chars %u ratio %.3f\n", p.size(), pj.n_structural_indexes, (double)pj.n_structural_indexes / p.size()); unsigned long total = cy1 + cy2 + cy3; printf( "stage 1 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: %10lu (failure %10lu)\n", cl1 / iterations, cy1 / iterations, 100. * cy1 / total, (double)cl1 / cy1, mis1/iterations, (double)cy1/mis1, cref1 / iterations, cmis1 / iterations); printf(" stage 1 runs at %.2f cycles per input byte.\n", (double)cy1 / (iterations * p.size())); printf( "stage 2 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: %10lu (failure %10lu)\n", cl2 / iterations, cy2 / iterations, 100. * cy2 / total, (double)cl2 / cy2, mis2/iterations, (double)cy2/mis2, cref2 /iterations, cmis2 / iterations); printf(" stage 2 runs at %.2f cycles per input byte and ", (double)cy2 / (iterations * p.size())); printf("%.2f cycles per structural character.\n", (double)cy2 / (iterations * pj.n_structural_indexes)); printf( "stage 3 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: %10lu (failure %10lu)\n", cl3 / iterations, cy3 /iterations, 100. * cy3 / total, (double)cl3 / cy3, mis3/iterations, (double)cy3/mis3, cref3 / iterations, cmis3 / iterations); printf(" stage 3 runs at %.2f cycles per input byte and ", (double)cy3 / (iterations * p.size())); printf("%.2f cycles per structural character.\n", (double)cy3 / (iterations * pj.n_structural_indexes)); printf(" all stages: %.2f cycles per input byte.\n", (double)total / (iterations * p.size())); #endif double min_result = *min_element(res.begin(), res.end()); cout << "Min: " << min_result << " bytes read: " << p.size() << " Gigabytes/second: " << (p.size()) / (min_result * 1000000000.0) << "\n"; if(dump) pj.printjson(); if (!isok) { printf(" Parsing failed. \n "); return EXIT_FAILURE; } return EXIT_SUCCESS; }