From 9f26355fe0596ee4524cab9b7374453a445fddd0 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sun, 25 Aug 2019 09:33:00 -0400 Subject: [PATCH] This should lower false positives. (#299) --- benchmark/perfdiff.cpp | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/benchmark/perfdiff.cpp b/benchmark/perfdiff.cpp index 6973a79c..2842dfa3 100644 --- a/benchmark/perfdiff.cpp +++ b/benchmark/perfdiff.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #ifdef _WIN32 #define popen _popen @@ -45,36 +47,42 @@ double readThroughput(std::string parseOutput) { } const double INTERLEAVED_ATTEMPTS = 7; -const double PERCENT_DIFFERENCE_THRESHOLD = -0.5; int main(int argc, char *argv[]) { if (argc != 3) { std::cerr << "Usage: " << argv[0] << " " << std::endl; return 1; } + std::vector ref; + std::vector newcode; for (int attempt=0; attempt < INTERLEAVED_ATTEMPTS; attempt++) { - if (attempt > 0) { - std::cout << "Running again to check whether it's a fluke ..." << std::endl; - } - std::cout << "Attempt #" << (attempt+1) << " of up to " << INTERLEAVED_ATTEMPTS << std::endl; + std::cout << "Attempt #" << (attempt+1) << " of " << INTERLEAVED_ATTEMPTS << std::endl; // Read new throughput double newThroughput = readThroughput(exec(argv[1])); std::cout << "New throughput: " << newThroughput << std::endl; + newcode.push_back(newThroughput); // Read reference throughput double referenceThroughput = readThroughput(exec(argv[2])); std::cout << "Ref throughput: " << referenceThroughput << std::endl; - - // Check if % difference > 0 - double percentDifference = ((newThroughput / referenceThroughput) - 1.0) * 100; - std::cout << "Difference: " << percentDifference << "%" << std::endl; - if (percentDifference >= PERCENT_DIFFERENCE_THRESHOLD) { - std::cout << "New throughput is same or better." << std::endl; - return 0; - } else { - std::cout << "New throughput is lower!"; - } + ref.push_back(referenceThroughput); } - return 1; + // we check if the maximum of newcode is lower than minimum of ref, if so we have a problem so fail! + double worseref = *std::min_element(ref.begin(), ref.end()); + double bestnewcode = *std::max_element(newcode.begin(), newcode.end()); + double bestref = *std::max_element(ref.begin(), ref.end()); + double worsenewcode = *std::min_element(newcode.begin(), newcode.end()); + std::cout << "The new code has a throughput in " << worsenewcode << " -- " << bestnewcode << std::endl; + std::cout << "The reference code has a throughput in " << worseref << " -- " << bestref << std::endl; + if(bestnewcode < worseref) { + std::cerr << "You probably have a performance degradation." << std::endl; + return EXIT_FAILURE; + } + if(bestnewcode < worseref) { + std::cout << "You probably have a performance gain." << std::endl; + return EXIT_SUCCESS; + } + std::cout << "There is no obvious performance difference. A manual check might be needed." << std::endl; + return EXIT_SUCCESS; }