jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

This should lower false positives. (#299)

This commit is contained in:
Daniel Lemire 2019-08-25 09:33:00 -04:00 committed by GitHub
parent f667d4965d
commit 9f26355fe0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
benchmark/perfdiff.cpp
View File

@ -5,6 +5,8 @@
#include <string> #include <string>
#include <sstream> #include <sstream>
#include <array> #include <array>
#include <algorithm>
#include <vector>
#ifdef _WIN32 #ifdef _WIN32
#define popen _popen #define popen _popen
@ -45,36 +47,42 @@ double readThroughput(std::string parseOutput) {
} }
const double INTERLEAVED_ATTEMPTS = 7; const double INTERLEAVED_ATTEMPTS = 7;
const double PERCENT_DIFFERENCE_THRESHOLD = -0.5;
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
if (argc != 3) { if (argc != 3) {
std::cerr << "Usage: " << argv[0] << " <new parse cmd> <reference parse cmd>" << std::endl; std::cerr << "Usage: " << argv[0] << " <new parse cmd> <reference parse cmd>" << std::endl;
return 1; return 1;
} }
std::vector<double> ref;
std::vector<double> newcode;
for (int attempt=0; attempt < INTERLEAVED_ATTEMPTS; attempt++) { for (int attempt=0; attempt < INTERLEAVED_ATTEMPTS; attempt++) {
if (attempt > 0) { std::cout << "Attempt #" << (attempt+1) << " of " << INTERLEAVED_ATTEMPTS << std::endl;
std::cout << "Running again to check whether it's a fluke ..." << std::endl;
}
std::cout << "Attempt #" << (attempt+1) << " of up to " << INTERLEAVED_ATTEMPTS << std::endl;
// Read new throughput // Read new throughput
double newThroughput = readThroughput(exec(argv[1])); double newThroughput = readThroughput(exec(argv[1]));
std::cout << "New throughput: " << newThroughput << std::endl; std::cout << "New throughput: " << newThroughput << std::endl;
newcode.push_back(newThroughput);
// Read reference throughput // Read reference throughput
double referenceThroughput = readThroughput(exec(argv[2])); double referenceThroughput = readThroughput(exec(argv[2]));
std::cout << "Ref throughput: " << referenceThroughput << std::endl; std::cout << "Ref throughput: " << referenceThroughput << std::endl;
ref.push_back(referenceThroughput);
// Check if % difference > 0
double percentDifference = ((newThroughput / referenceThroughput) - 1.0) * 100;
std::cout << "Difference: " << percentDifference << "%" << std::endl;
if (percentDifference >= PERCENT_DIFFERENCE_THRESHOLD) {
std::cout << "New throughput is same or better." << std::endl;
return 0;
} else {
std::cout << "New throughput is lower!";
}
} }
return 1; // we check if the maximum of newcode is lower than minimum of ref, if so we have a problem so fail!
double worseref = *std::min_element(ref.begin(), ref.end());
double bestnewcode = *std::max_element(newcode.begin(), newcode.end());
double bestref = *std::max_element(ref.begin(), ref.end());
double worsenewcode = *std::min_element(newcode.begin(), newcode.end());
std::cout << "The new code has a throughput in " << worsenewcode << " -- " << bestnewcode << std::endl;
std::cout << "The reference code has a throughput in " << worseref << " -- " << bestref << std::endl;
if(bestnewcode < worseref) {
std::cerr << "You probably have a performance degradation." << std::endl;
return EXIT_FAILURE;
}
if(bestnewcode < worseref) {
std::cout << "You probably have a performance gain." << std::endl;
return EXIT_SUCCESS;
}
std::cout << "There is no obvious performance difference. A manual check might be needed." << std::endl;
return EXIT_SUCCESS;
} }