This should lower false positives. (#299)

benchmark/perfdiff.cpp

@@ -5,6 +5,8 @@
 #include <string>
 #include <sstream>
 #include <array>
+#include <algorithm>
+#include <vector>
 
 #ifdef _WIN32
 #define popen _popen
@@ -45,36 +47,42 @@ double readThroughput(std::string parseOutput) {
 }
 
 const double INTERLEAVED_ATTEMPTS = 7;
-const double PERCENT_DIFFERENCE_THRESHOLD = -0.5;
 
 int main(int argc, char *argv[]) {
     if (argc != 3) {
         std::cerr << "Usage: " << argv[0] << " <new parse cmd> <reference parse cmd>" << std::endl;
         return 1;
     }
+    std::vector<double> ref;
+    std::vector<double> newcode;
     for (int attempt=0; attempt < INTERLEAVED_ATTEMPTS; attempt++) {
-        if (attempt > 0) {
-            std::cout << "Running again to check whether it's a fluke ..." << std::endl;
-        }
-        std::cout << "Attempt #" << (attempt+1) << " of up to " << INTERLEAVED_ATTEMPTS << std::endl;
+        std::cout << "Attempt #" << (attempt+1) << " of " << INTERLEAVED_ATTEMPTS << std::endl;
 
         // Read new throughput
         double newThroughput = readThroughput(exec(argv[1]));
         std::cout << "New throughput: " << newThroughput << std::endl;
+        newcode.push_back(newThroughput);
 
         // Read reference throughput
         double referenceThroughput = readThroughput(exec(argv[2]));
         std::cout << "Ref throughput: " << referenceThroughput << std::endl;
-
-        // Check if % difference > 0
-        double percentDifference = ((newThroughput / referenceThroughput) - 1.0) * 100;
-        std::cout << "Difference: " << percentDifference << "%" << std::endl;
-        if (percentDifference >= PERCENT_DIFFERENCE_THRESHOLD) {
-            std::cout << "New throughput is same or better." << std::endl;
-            return 0;
-        } else {
-            std::cout << "New throughput is lower!";
-        }
+        ref.push_back(referenceThroughput);
     }
-    return 1;
+    // we check if the maximum of newcode is lower than minimum of ref, if so we have a problem so fail!
+    double worseref = *std::min_element(ref.begin(), ref.end());
+    double bestnewcode =  *std::max_element(newcode.begin(), newcode.end());
+    double bestref = *std::max_element(ref.begin(), ref.end());
+    double worsenewcode =  *std::min_element(newcode.begin(), newcode.end());
+    std::cout << "The new code has a throughput in       " << worsenewcode << " -- " << bestnewcode << std::endl;
+    std::cout << "The reference code has a throughput in " << worseref << " -- " << bestref << std::endl;
+    if(bestnewcode < worseref) {
+      std::cerr << "You probably have a performance degradation." << std::endl;
+      return EXIT_FAILURE;
+    }
+    if(bestnewcode < worseref) {
+      std::cout << "You probably have a performance gain." << std::endl;
+      return EXIT_SUCCESS;
+    }
+    std::cout << "There is no obvious performance difference. A manual check might be needed." << std::endl;
+    return EXIT_SUCCESS;
 }