Pave the way for non-record-based benchmarks

benchmark/json_benchmark.h

@@ -1,31 +1,16 @@
 #pragma once
 
-template<typename B, typename R> static void ParseRecordsBenchmark(benchmark::State &state, const simdjson::padded_string &json) {
+template<typename B, typename R> static void JsonBenchmark(benchmark::State &state, const simdjson::padded_string &json) {
   event_collector collector(true);
   event_aggregate events;
 
-
   // Warmup and equality check (make sure the data is right!)
   B bench;
-  bench.SetUp();
   if (!bench.Run(json)) { state.SkipWithError("warmup tweet reading failed"); return; }
   {
     R reference;
-    reference.SetUp();
     if (!reference.Run(json)) { state.SkipWithError("reference tweet reading failed"); return; }
-    assert(bench.Records().size() == reference.Records().size());
+    assert(bench.Result() == reference.Result());
-    for (size_t i=0; i<bench.Records().size(); i++) {
-      if (bench.Records()[i] != reference.Records()[i]) {
-        std::cerr << "Bench Record " << i << std::endl;
-        std::cerr << "----------------------" << std::endl;
-        std::cerr << bench.Records()[i] << std::endl;
-        std::cerr << "Reference Record " << i << std::endl;
-        std::cerr << "----------------------" << std::endl;
-        std::cerr << reference.Records()[i] << std::endl;
-        throw "Parse produced the wrong values!";
-      }
-    }
-    reference.TearDown();
   }
 
   // Run the benchmark
@@ -38,9 +23,9 @@ template<typename B, typename R> static void ParseRecordsBenchmark(benchmark::St
   }
 
   state.SetBytesProcessed(json.size() * state.iterations());
-  state.SetItemsProcessed(bench.Records().size() * state.iterations());
+  state.SetItemsProcessed(bench.ItemCount() * state.iterations());
   state.counters["best_bytes_per_sec"] = benchmark::Counter(double(json.size()) / events.best.elapsed_sec());
-  state.counters["best_items_per_sec"] = benchmark::Counter(double(bench.Records().size()) / events.best.elapsed_sec());
+  state.counters["best_items_per_sec"] = benchmark::Counter(double(bench.ItemCount()) / events.best.elapsed_sec());
 
   state.counters["docs_per_sec"] = benchmark::Counter(1.0, benchmark::Counter::kIsIterationInvariantRate);
   state.counters["best_docs_per_sec"] = benchmark::Counter(1.0 / events.best.elapsed_sec());
@@ -69,7 +54,7 @@ template<typename B, typename R> static void ParseRecordsBenchmark(benchmark::St
     state.counters["best_frequency"]              = events.best.cycles()       / events.best.elapsed_sec();
   }
   state.counters["bytes"] = benchmark::Counter(double(json.size()));
-  state.counters["items"] = benchmark::Counter(double(bench.Records().size()));
+  state.counters["items"] = benchmark::Counter(double(bench.ItemCount()));
 
   // Build the label
   using namespace std;
@@ -87,7 +72,7 @@ template<typename B, typename R> static void ParseRecordsBenchmark(benchmark::St
     label << " cache_ref="    << setw(10) << uint64_t(events.best.cache_references()) << setw(0);
   }
 
-  label << " items=" << setw(10) << bench.Records().size() << setw(0);
+  label << " items=" << setw(10) << bench.ItemCount() << setw(0);
   label << " avg_time=" << setw(10) << uint64_t(events.elapsed_ns()) << setw(0) << " ns";
   label << "]";
 

benchmark/largerandom/dom.h

@@ -11,9 +11,9 @@ using namespace simdjson;
 class Dom {
 public:
   simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline bool SetUp() { return true; }
+
-  simdjson_really_inline bool TearDown() { return true; }
+  simdjson_really_inline const std::vector<my_point> &Result() { return container; }
-  simdjson_really_inline const std::vector<my_point> &Records() { return container; }
+  simdjson_really_inline size_t ItemCount() { return container.size(); }
 
 private:
   dom::parser parser{};

benchmark/largerandom/iter.h

@@ -15,9 +15,9 @@ using namespace SIMDJSON_IMPLEMENTATION::stage2;
 class Iter {
 public:
   simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline bool SetUp() { return true; }
+
-  simdjson_really_inline bool TearDown() { return true; }
+  simdjson_really_inline const std::vector<my_point> &Result() { return container; }
-  simdjson_really_inline const std::vector<my_point> &Records() { return container; }
+  simdjson_really_inline size_t ItemCount() { return container.size(); }
 
 private:
   ondemand::parser parser{};

benchmark/largerandom/largerandom.h

@@ -39,13 +39,13 @@ struct my_point {
   double x;
   double y;
   double z;
-  bool operator==(const my_point &other) const {
+  simdjson_really_inline bool operator==(const my_point &other) const {
     return x == other.x && y == other.y && z == other.z;
   }
-  bool operator!=(const my_point &other) const { return !(*this == other); }
+  simdjson_really_inline bool operator!=(const my_point &other) const { return !(*this == other); }
 };
 
-static std::ostream &operator<<(std::ostream &o, const my_point &p) {
+SIMDJSON_UNUSED static std::ostream &operator<<(std::ostream &o, const my_point &p) {
   return o << p.x << "," << p.y << "," << p.z << std::endl;
 }
 
@@ -57,12 +57,12 @@ static std::ostream &operator<<(std::ostream &o, const my_point &p) {
 #include <vector>
 #include "event_counter.h"
 #include "dom.h"
-#include "parse_records_benchmark.h"
+#include "json_benchmark.h"
 
 namespace largerandom {
 
 template<typename T> static void LargeRandom(benchmark::State &state) {
-  ParseRecordsBenchmark<T, Dom>(state, get_built_json_array());
+  JsonBenchmark<T, Dom>(state, get_built_json_array());
 }
 
 } // namespace largerandom

benchmark/largerandom/ondemand.h

@@ -15,9 +15,9 @@ using namespace SIMDJSON_IMPLEMENTATION::stage2;
 class OnDemand {
 public:
   simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline bool SetUp() { return true; }
+
-  simdjson_really_inline bool TearDown() { return true; }
+  simdjson_really_inline const std::vector<my_point> &Result() { return container; }
-  simdjson_really_inline const std::vector<my_point> &Records() { return container; }
+  simdjson_really_inline size_t ItemCount() { return container.size(); }
 
 private:
   ondemand::parser parser{};

benchmark/largerandom/sax.h

@@ -15,9 +15,10 @@ using namespace SIMDJSON_IMPLEMENTATION::stage2;
 class Sax {
 public:
   simdjson_really_inline bool Run(const padded_string &json) noexcept;
-  simdjson_really_inline bool SetUp() noexcept { return true; }
+
-  simdjson_really_inline bool TearDown() noexcept { return true; }
+  simdjson_really_inline const std::vector<my_point> &Result() { return container; }
-  simdjson_really_inline const std::vector<my_point> &Records() { return container; }
+  simdjson_really_inline size_t ItemCount() { return container.size(); }
+
 private:
   simdjson_really_inline error_code RunNoExcept(const padded_string &json) noexcept;
   error_code Allocate(size_t new_capacity);

benchmark/partial_tweets/dom.h

@@ -11,9 +11,9 @@ using namespace simdjson;
 class Dom {
 public:
   simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline bool SetUp() { return true; }
+
-  simdjson_really_inline bool TearDown() { return true; }
+  simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
-  simdjson_really_inline const std::vector<tweet> &Records() { return tweets; }
+  simdjson_really_inline size_t ItemCount() { return tweets.size(); }
 
 private:
   dom::parser parser{};

benchmark/partial_tweets/domnoexcept.h

@@ -13,9 +13,9 @@ using namespace SIMDJSON_IMPLEMENTATION::stage2;
 class DomNoExcept {
 public:
   simdjson_really_inline bool Run(const simdjson::padded_string &json) noexcept;
-  simdjson_really_inline bool SetUp() noexcept { return true; }
+
-  simdjson_really_inline bool TearDown() noexcept { return true; }
+  simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
-  simdjson_really_inline const std::vector<tweet> &Records() { return tweets; }
+  simdjson_really_inline size_t ItemCount() { return tweets.size(); }
 
 private:
   dom::parser parser{};

benchmark/partial_tweets/iter.h

@@ -15,9 +15,9 @@ using namespace SIMDJSON_IMPLEMENTATION::stage2;
 class Iter {
 public:
   simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline bool SetUp() { return true; }
+
-  simdjson_really_inline bool TearDown() { return true; }
+  simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
-  simdjson_really_inline const std::vector<tweet> &Records() { return tweets; }
+  simdjson_really_inline size_t ItemCount() { return tweets.size(); }
 
 private:
   ondemand::parser parser{};

benchmark/partial_tweets/ondemand.h

@@ -15,9 +15,9 @@ using namespace SIMDJSON_IMPLEMENTATION::stage2;
 class OnDemand {
 public:
   simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline bool SetUp() { return true; }
+
-  simdjson_really_inline bool TearDown() { return true; }
+  simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
-  simdjson_really_inline const std::vector<tweet> &Records() { return tweets; }
+  simdjson_really_inline size_t ItemCount() { return tweets.size(); }
 
 private:
   ondemand::parser parser{};

benchmark/partial_tweets/partial_tweets.h

@@ -16,7 +16,7 @@ template<typename T> static void PartialTweets(benchmark::State &state);
 #include <vector>
 #include "event_counter.h"
 #include "domnoexcept.h"
-#include "parse_records_benchmark.h"
+#include "json_benchmark.h"
 
 namespace partial_tweets {
 
@@ -35,7 +35,7 @@ template<typename T> static void PartialTweets(benchmark::State &state) {
     return;
   }
 
-  ParseRecordsBenchmark<T, DomNoExcept>(state, json);
+  JsonBenchmark<T, DomNoExcept>(state, json);
 }
 
 } // namespace partial_tweets

benchmark/partial_tweets/sax.h

@@ -15,9 +15,10 @@ using namespace SIMDJSON_IMPLEMENTATION::stage2;
 class Sax {
 public:
   simdjson_really_inline bool Run(const padded_string &json) noexcept;
-  simdjson_really_inline bool SetUp() noexcept { return true; }
+
-  simdjson_really_inline bool TearDown() noexcept { return true; }
+  simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
-  simdjson_really_inline const std::vector<tweet> &Records() { return tweets; }
+  simdjson_really_inline size_t ItemCount() { return tweets.size(); }
+
 private:
   simdjson_really_inline error_code RunNoExcept(const padded_string &json) noexcept;
   error_code Allocate(size_t new_capacity);

benchmark/partial_tweets/tweet.h

@@ -30,7 +30,7 @@ struct tweet {
   twitter_user user{};
   uint64_t retweet_count{};
   uint64_t favorite_count{};
-  bool operator==(const tweet &other) const {
+  simdjson_really_inline bool operator==(const tweet &other) const {
     return created_at == other.created_at &&
            id == other.id &&
            text == other.text &&
@@ -39,10 +39,10 @@ struct tweet {
            retweet_count == other.retweet_count &&
            favorite_count == other.favorite_count;
   }
-  bool operator!=(const tweet &other) const { return !(*this == other); }
+  simdjson_really_inline bool operator!=(const tweet &other) const { return !(*this == other); }
 };
 
-static std::ostream &operator<<(std::ostream &o, const tweet &t) {
+SIMDJSON_UNUSED static std::ostream &operator<<(std::ostream &o, const tweet &t) {
   o << "created_at: " << t.created_at << std::endl;
   o << "id: " << t.id << std::endl;
   o << "text: " << t.text << std::endl;