#include "jsonparser/common_defs.h" #include "linux-perf-events.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include //#define DEBUG #include "jsonparser/jsonparser.h" #include "jsonparser/jsonioutil.h" #include "jsonparser/simdjson_internal.h" #include "jsonparser/stage1_find_marks.h" #include "jsonparser/stage2_flatten.h" #include "jsonparser/stage34_unified.h" using namespace std; // https://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal namespace Color { enum Code { FG_DEFAULT = 39, FG_BLACK = 30, FG_RED = 31, FG_GREEN = 32, FG_YELLOW = 33, FG_BLUE = 34, FG_MAGENTA = 35, FG_CYAN = 36, FG_LIGHT_GRAY = 37, FG_DARK_GRAY = 90, FG_LIGHT_RED = 91, FG_LIGHT_GREEN = 92, FG_LIGHT_YELLOW = 93, FG_LIGHT_BLUE = 94, FG_LIGHT_MAGENTA = 95, FG_LIGHT_CYAN = 96, FG_WHITE = 97, BG_RED = 41, BG_GREEN = 42, BG_BLUE = 44, BG_DEFAULT = 49 }; class Modifier { Code code; public: Modifier(Code pCode) : code(pCode) {} friend std::ostream &operator<<(std::ostream &os, const Modifier &mod) { return os << "\033[" << mod.code << "m"; } }; } // namespace Color void colorfuldisplay(ParsedJson &pj, const u8 *buf) { Color::Modifier greenfg(Color::FG_GREEN); Color::Modifier yellowfg(Color::FG_YELLOW); Color::Modifier deffg(Color::FG_DEFAULT); size_t i = 0; // skip initial fluff while ((i + 1 < pj.n_structural_indexes) && (pj.structural_indexes[i] == pj.structural_indexes[i + 1])) { i++; } for (; i < pj.n_structural_indexes; i++) { u32 idx = pj.structural_indexes[i]; u8 c = buf[idx]; if (((c & 0xdf) == 0x5b)) { // meaning 7b or 5b, { or [ std::cout << greenfg << buf[idx] << deffg; } else if (((c & 0xdf) == 0x5d)) { // meaning 7d or 5d, } or ] std::cout << greenfg << buf[idx] << deffg; } else { std::cout << yellowfg << buf[idx] << deffg; } if (i + 1 < pj.n_structural_indexes) { u32 nextidx = pj.structural_indexes[i + 1]; for (u32 pos = idx + 1; pos < nextidx; pos++) { std::cout << buf[pos]; } } } std::cout << std::endl; } int main(int argc, char *argv[]) { bool verbose = false; bool dump = false; int c; while ((c = getopt (argc, argv, "vd")) != -1) switch (c) { case 'v': verbose = true; break; case 'd': dump = true; break; default: abort (); } if (optind >= argc) { cerr << "Usage: " << argv[0] << " " << endl; exit(1); } const char * filename = argv[optind]; if(optind + 1 < argc) { cerr << "warning: ignoring everything after " << argv[optind + 1] << endl; } if(verbose) cout << "[verbose] loading " << filename << endl; pair p = get_corpus(filename); if(verbose) cout << "[verbose] loaded " << filename << " ("<< p.second << " bytes)" << endl; ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); ParsedJson &pj(*pj_ptr); if(verbose) cout << "[verbose] allocated memory for parsed JSON " << endl; #if defined(DEBUG) const u32 iterations = 1; #else const u32 iterations = p.second < 1 * 1000 * 1000? 1000 : 10; #endif vector res; res.resize(iterations); #if !defined(__linux__) #define SQUASH_COUNTERS #endif #ifndef SQUASH_COUNTERS vector evts; evts.push_back(PERF_COUNT_HW_CPU_CYCLES); evts.push_back(PERF_COUNT_HW_INSTRUCTIONS); evts.push_back(PERF_COUNT_HW_BRANCH_MISSES); LinuxEvents unified(evts); vector results; results.resize(evts.size()); unsigned long cy1 = 0, cy2 = 0, cy3 = 0; unsigned long cl1 = 0, cl2 = 0, cl3 = 0; unsigned long mis1 = 0, mis2 = 0, mis3 = 0; #endif bool isok = true; for (u32 i = 0; i < iterations; i++) { if(verbose) cout << "[verbose] iteration # " << i << endl; auto start = std::chrono::steady_clock::now(); #ifndef SQUASH_COUNTERS unified.start(); #endif isok = find_structural_bits(p.first, p.second, pj); #ifndef SQUASH_COUNTERS unified.end(results); cy1 += results[0]; cl1 += results[1]; mis1 += results[2]; if (!isok) { cout << "Failed out during stage 1\n"; break; } unified.start(); #endif isok = flatten_indexes(p.second, pj); #ifndef SQUASH_COUNTERS unified.end(results); cy2 += results[0]; cl2 += results[1]; mis2 += results[2]; if (!isok) { cout << "Failed out during stage 2\n"; break; } unified.start(); #endif isok = unified_machine(p.first, p.second, pj); #ifndef SQUASH_COUNTERS unified.end(results); cy3 += results[0]; cl3 += results[1]; mis3 += results[2]; if (!isok) { cout << "Failed out during stage 34\n"; break; } #endif auto end = std::chrono::steady_clock::now(); std::chrono::duration secs = end - start; res[i] = secs.count(); } #ifndef SQUASH_COUNTERS printf("number of bytes %ld number of structural chars %d ratio %.3f\n", p.second, pj.n_structural_indexes, (double)pj.n_structural_indexes / p.second); unsigned long total = cy1 + cy2 + cy3; printf( "stage 1 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) \n", cl1 / iterations, cy1 / iterations, 100. * cy1 / total, (double)cl1 / cy1, mis1/iterations, (double)cy1/mis1); printf(" stage 1 runs at %.2f cycles per input byte.\n", (double)cy1 / (iterations * p.second)); printf( "stage 2 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) \n", cl2 / iterations, cy2 / iterations, 100. * cy2 / total, (double)cl2 / cy2, mis2/iterations, (double)cy2/mis2); printf(" stage 2 runs at %.2f cycles per input byte and ", (double)cy2 / (iterations * p.second)); printf("%.2f cycles per structural character.\n", (double)cy2 / (iterations * pj.n_structural_indexes)); printf( "stage 3 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f)\n", cl3 / iterations, cy3 /iterations, 100. * cy3 / total, (double)cl3 / cy3, mis3/iterations, (double)cy3/mis3); printf(" stage 3 runs at %.2f cycles per input byte and ", (double)cy3 / (iterations * p.second)); printf("%.2f cycles per structural character.\n", (double)cy3 / (iterations * pj.n_structural_indexes)); printf(" all stages: %.2f cycles per input byte.\n", (double)total / (iterations * p.second)); #endif // colorfuldisplay(pj, p.first); double min_result = *min_element(res.begin(), res.end()); cout << "Min: " << min_result << " bytes read: " << p.second << " Gigabytes/second: " << (p.second) / (min_result * 1000000000.0) << "\n"; if(dump) pj_ptr->dump_tapes(); free(p.first); deallocate_ParsedJson(pj_ptr); if (!isok) { printf(" Parsing failed. \n "); return EXIT_FAILURE; } return EXIT_SUCCESS; }