jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Merge pull request #857 from simdjson/jkeiser/benchfeatures 

Make benchfeatures work again
This commit is contained in:
John Keiser 2020-05-05 12:34:49 -07:00 committed by GitHub
parent 25fe6d7dde 1e8a54af0b
commit a64d2f4673
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 95 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -87,15 +87,12 @@ objs
# Build outputs
/build*/
/visual_studio/
# Fuzzer outputs generated by instructions in fuzz/Fuzzing.md
/corpus.zip
/ossfuzz-out/
/out/
# Don't check in generated API docs
# Generated docs
/doc/api
# Don't check in generated examples
/jsonexamples/generated
/visual_studio

19
benchmark/CMakeLists.txt
View File

@ -1,6 +1,6 @@
include_directories( . linux )
link_libraries(simdjson simdjson-windows-headers)
# add_executable(benchfeatures benchfeatures.cpp) # doesn't presently compile at all
link_libraries(simdjson simdjson-flags simdjson-windows-headers test-data)
add_executable(benchfeatures benchfeatures.cpp)
add_executable(get_corpus_benchmark get_corpus_benchmark.cpp)
add_executable(perfdiff perfdiff.cpp)
add_executable(parse parse.cpp)
@ -18,8 +18,6 @@ if (SIMDJSON_GOOGLE_BENCHMARKS)
link_libraries(benchmark::benchmark)
add_executable(bench_parse_call bench_parse_call.cpp)
add_executable(bench_dom_api bench_dom_api.cpp)
target_link_libraries(bench_dom_api test-data)
target_link_libraries(bench_parse_call test-data)
endif()
if (SIMDJSON_COMPETITION)
@ -37,16 +35,3 @@ if (SIMDJSON_COMPETITION)
endif()
include(checkperf.cmake)
# IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
# add_test(NAME checkperf
# COMMAND ${CMAKE_COMMAND} -E env
# CHECKPERF_REPOSITORY=${SIMDJSON_GITHUB_REPOSITORY}
# CHECKPERF_BRANCH=master
# CHECKPERF_DIR=${CMAKE_CURRENT_BINARY_DIR}/simdjson-master
# CHECKPERF_CMAKECACHE=${SIMDJSON_USER_CMAKECACHE}
# bash ${CMAKE_CURRENT_SOURCE_DIR}/checkperf.sh ${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json)
# set_property(TEST checkperf APPEND PROPERTY LABELS per_implementation)
# set_property(TEST checkperf APPEND PROPERTY DEPENDS parse perfdiff ${SIMDJSON_USER_CMAKECACHE})
# set_property(TEST checkperf PROPERTY RUN_SERIAL TRUE)
# ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")

110
benchmark/benchfeatures.cpp
View File

@ -75,7 +75,6 @@ void exit_usage(string message) {
}
struct option_struct {
architecture arch = architecture::UNSUPPORTED;
bool stage1_only = false;
int32_t iterations = 400;
@ -98,10 +97,7 @@ struct option_struct {
verbose = true;
break;
case 'a':
arch = parse_architecture(optarg);
if (arch == architecture::UNSUPPORTED) {
exit_usage(string("Unsupported option value -a ") + optarg + ": expected -a HASWELL, WESTMERE or ARM64");
}
simdjson::active_implementation = simdjson::available_implementations[optarg];
break;
case 's':
if (!strcmp(optarg, "stage1")) {
@ -113,15 +109,9 @@ struct option_struct {
}
break;
default:
exit_error("Unexpected argument " + c);
exit_error(string("Unexpected argument ") + std::string(1,static_cast<char>(c)));
}
}
// If architecture is not specified, pick the best supported architecture by default
if (arch == architecture::UNSUPPORTED) {
arch = find_best_supported_architecture();
}
dom::parser::use_implementation(arch);
}
template<typename F>
@ -150,20 +140,20 @@ struct feature_benchmarker {
benchmarker struct23;
benchmarker struct23_miss;
feature_benchmarker(const simdjson::implementation &parser, event_collector& collector) :
utf8 ("jsonexamples/generated/utf-8.json", parser, collector),
utf8_miss ("jsonexamples/generated/utf-8-miss.json", parser, collector),
escape ("jsonexamples/generated/escape.json", parser, collector),
escape_miss ("jsonexamples/generated/escape-miss.json", parser, collector),
empty ("jsonexamples/generated/0-structurals.json", parser, collector),
empty_miss ("jsonexamples/generated/0-structurals-miss.json", parser, collector),
struct7 ("jsonexamples/generated/7-structurals.json", parser, collector),
struct7_miss ("jsonexamples/generated/7-structurals-miss.json", parser, collector),
struct7_full ("jsonexamples/generated/7-structurals-full.json", parser, collector),
struct15 ("jsonexamples/generated/15-structurals.json", parser, collector),
struct15_miss("jsonexamples/generated/15-structurals-miss.json", parser, collector),
struct23 ("jsonexamples/generated/23-structurals.json", parser, collector),
struct23_miss("jsonexamples/generated/23-structurals-miss.json", parser, collector)
feature_benchmarker(event_collector& collector) :
utf8 (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8.json", collector),
utf8_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8-miss.json", collector),
escape (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape.json", collector),
escape_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape-miss.json", collector),
empty (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals.json", collector),
empty_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals-miss.json", collector),
struct7 (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals.json", collector),
struct7_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-miss.json", collector),
struct7_full (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-full.json", collector),
struct15 (SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals.json", collector),
struct15_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals-miss.json", collector),
struct23 (SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals.json", collector),
struct23_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals-miss.json", collector)
{
}
@ -185,7 +175,7 @@ struct feature_benchmarker {
}
double cost_per_block(BenchmarkStage stage, const benchmarker& feature, size_t feature_blocks, const benchmarker& base) const {
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / feature_blocks;
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / double(feature_blocks);
}
// Whether we're recording cache miss and branch miss events
@ -195,7 +185,7 @@ struct feature_benchmarker {
// Base cost of any block (including empty ones)
double base_cost(BenchmarkStage stage) const {
return (empty[stage].best.elapsed_ns() / empty.stats->blocks);
return (empty[stage].best.elapsed_ns() / double(empty.stats->blocks));
}
// Extra cost of a 1-7 structural block over an empty block
@ -209,7 +199,7 @@ struct feature_benchmarker {
// Rate of 1-7-structural misses per 8-structural flip
double struct1_7_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses()) / struct7_miss.stats->blocks_with_1_structural_flipped;
return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
}
// Extra cost of an 8-15 structural block over a 1-7 structural block
@ -223,7 +213,7 @@ struct feature_benchmarker {
// Rate of 8-15-structural misses per 8-structural flip
double struct8_15_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / struct15_miss.stats->blocks_with_8_structurals_flipped;
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
}
// Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
@ -237,7 +227,7 @@ struct feature_benchmarker {
// Rate of 16-structural misses per 16-structural flip
double struct16_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / struct23_miss.stats->blocks_with_16_structurals_flipped;
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
}
// Extra cost of having UTF-8 in a block
@ -251,7 +241,7 @@ struct feature_benchmarker {
// Rate of UTF-8 misses per UTF-8 flip
double utf8_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / utf8_miss.stats->blocks_with_utf8_flipped;
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
}
// Extra cost of having escapes in a block
@ -265,39 +255,39 @@ struct feature_benchmarker {
// Rate of escape misses per escape flip
double escape_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / escape_miss.stats->blocks_with_escapes_flipped;
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
}
double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
// Expected base ns/block (empty)
json_stats& stats = *file.stats;
double expected = base_cost(stage) * stats.blocks;
expected += struct1_7_cost(stage) * stats.blocks_with_1_structural;
expected += utf8_cost(stage) * stats.blocks_with_utf8;
expected += escape_cost(stage) * stats.blocks_with_escapes;
expected += struct8_15_cost(stage) * stats.blocks_with_8_structurals;
expected += struct16_cost(stage) * stats.blocks_with_16_structurals;
return expected / stats.blocks;
double expected = base_cost(stage) * double(stats.blocks);
expected += struct1_7_cost(stage) * double(stats.blocks_with_1_structural);
expected += utf8_cost(stage) * double(stats.blocks_with_utf8);
expected += escape_cost(stage) * double(stats.blocks_with_escapes);
expected += struct8_15_cost(stage) * double(stats.blocks_with_8_structurals);
expected += struct16_cost(stage) * double(stats.blocks_with_16_structurals);
return expected / double(stats.blocks);
}
double calc_expected_miss_cost(BenchmarkStage stage, const benchmarker& file) const {
// Expected base ns/block (empty)
json_stats& stats = *file.stats;
double expected = struct1_7_miss_cost(stage) * stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
expected += utf8_miss_cost(stage) * stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
expected += escape_miss_cost(stage) * stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
expected += struct8_15_miss_cost(stage) * stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
expected += struct16_miss_cost(stage) * stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
return expected / stats.blocks;
double expected = struct1_7_miss_cost(stage) * double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
expected += utf8_miss_cost(stage) * double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
expected += escape_miss_cost(stage) * double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
expected += struct8_15_miss_cost(stage) * double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
expected += struct16_miss_cost(stage) * double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
return expected / double(stats.blocks);
}
double calc_expected_misses(BenchmarkStage stage, const benchmarker& file) const {
json_stats& stats = *file.stats;
double expected = stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
expected += stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
expected += stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
expected += stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
expected += stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
double expected = double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
expected += double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
expected += double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
expected += double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
expected += double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
return expected;
}
@ -364,10 +354,10 @@ struct feature_benchmarker {
};
void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
double actual = results[stage].best.elapsed_ns() / results.stats->blocks;
double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
double calc = features.calc_expected(stage, results);
uint64_t actual_misses = results[stage].best.branch_misses();
uint64_t calc_misses = uint64_t(features.calc_expected_misses(stage, results));
double actual_misses = results[stage].best.branch_misses();
double calc_misses = features.calc_expected_misses(stage, results);
double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
printf(" | %-8s ", benchmark_stage_name(stage));
printf("| %-15s ", filename);
@ -376,10 +366,10 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
printf("| %8.3g ", calc);
printf("| %8.3g ", actual);
printf("| %+8.3g ", actual - calc);
printf("| %13lu ", calc_misses);
printf("| %13llu ", (long long unsigned)(calc_misses));
if (features.has_events()) {
printf("| %13lu ", actual_misses);
printf("| %+13ld ", int64_t(actual_misses - calc_misses));
printf("| %13llu ", (long long unsigned)(actual_misses));
printf("| %+13lld ", (long long int)(actual_misses - calc_misses));
double miss_adjustment = calc_miss_cost * (double(int64_t(actual_misses - calc_misses)) / calc_misses);
printf("| %8.3g ", calc_miss_cost + miss_adjustment);
printf("| %+8.3g ", actual - (calc + miss_adjustment));
@ -401,9 +391,9 @@ int main(int argc, char *argv[]) {
// Set up benchmarkers by reading all files
feature_benchmarker features(collector);
benchmarker gsoc_2018("jsonexamples/gsoc-2018.json", collector);
benchmarker twitter("jsonexamples/twitter.json", collector);
benchmarker random("jsonexamples/random.json", collector);
benchmarker gsoc_2018(SIMDJSON_BENCHMARK_DATA_DIR "gsoc-2018.json", collector);
benchmarker twitter(SIMDJSON_BENCHMARK_DATA_DIR "twitter.json", collector);
benchmarker random(SIMDJSON_BENCHMARK_DATA_DIR "random.json", collector);
// Run the benchmarks
progress_bar progress(options.iterations, 100);

19
jsonexamples/CMakeLists.txt
View File

@ -1,5 +1,16 @@
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/twitter.json PARENT_SCOPE)
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_SOURCE_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
set(EXAMPLE_JSON ${CMAKE_CURRENT_BINARY_DIR}/twitter.json PARENT_SCOPE)
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_BINARY_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
# Copy static files to the build dir so they live alongside the generated ones
file(GLOB_RECURSE example_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.json *.ndjson)
foreach(example_file ${example_files})
configure_file(${example_file} ${example_file} COPYONLY)
endforeach(example_file)
add_subdirectory(generated)
add_library(jsonexamples-data INTERFACE)
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_BINARY_DIR}/")
add_dependencies(jsonexamples-data ${example_files})
add_dependencies(jsonexamples-data generated-data)

17
jsonexamples/generated/CMakeLists.txt Normal file
View File

@ -0,0 +1,17 @@
set(generated_files
utf-8.json escape.json
0-structurals.json 7-structurals.json 15-structurals.json 23-structurals.json
)
find_package(Ruby QUIET)
if (RUBY_EXECUTABLE)
add_custom_command(
OUTPUT ${generated_files}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/miss-templates/*.txt
COMMAND ${RUBY_EXECUTABLE} genfeaturejson.rb ${CMAKE_CURRENT_BINARY_DIR}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)
add_custom_target(generated-data DEPENDS ${generated_files})
else (RUBY_EXECUTABLE)
# TODO make this work without Ruby and make it part of the normal build
add_custom_target(generated-data)
endif (RUBY_EXECUTABLE)

16
benchmark/genfeaturejson.rb → jsonexamples/generated/genfeaturejson.rb
View File

@ -1,3 +1,5 @@
# encoding: utf-8
class ChunkWriter
def initialize(output_dir, miss_templates, file_size=640*1000, block_size=64)
@@output_dir = output_dir
@ -12,7 +14,7 @@ class ChunkWriter
end.join("")
end
def write_files(filename, start1, repeat1, end1, repeat2: '', include_newline: true)
def write_files(filename, start1, repeat1, end1, repeat2='', include_newline=true)
start1 = prepare_chunk(start1, include_newline)
repeat1 = prepare_chunk(repeat1, include_newline)
end1 = prepare_chunk(end1, include_newline)
@ -26,7 +28,7 @@ class ChunkWriter
def write_full(filename, start1, repeat1, end1)
puts "Writing #{filename} ..."
File.open(filename, "w") do |file|
File.open(filename, "wb") do |file|
write_chunks(file, start1, repeat1, end1, @@file_size)
end
raise "OMG wrong file size #{File.size(filename)} (should be #{@@file_size})" if File.size(filename) != @@file_size
@ -39,7 +41,7 @@ class ChunkWriter
halfway_point = start1.bytesize + repeat1_len + repeat2.bytesize
puts "Writing #{filename} ..."
File.open(filename, "w") do |file|
File.open(filename, "wb") do |file|
write_chunks(file, start1, repeat1, repeat2, halfway_point)
write_chunks(file, repeat2, repeat2, end1, @@file_size-halfway_point)
end
@ -55,7 +57,7 @@ class ChunkWriter
repeat_template = (repeat_chunks - 1).step(repeat_template.size - 1, repeat_chunks).map { |i| repeat_template[i] }
puts "Writing #{filename} ..."
File.open(filename, "w") do |file|
File.open(filename, "wb") do |file|
file.write(start1)
repeat_template.each do |should_repeat|
file.write(should_repeat == "1" ? repeat1 : repeat2)
@ -83,12 +85,12 @@ class ChunkWriter
end
end
output_dir = File.expand_path("../jsonexamples/generated", File.dirname(__FILE__))
output_dir = ARGV[0] || File.expand_path(".", File.dirname(__FILE__))
miss_templates = File.expand_path("miss-templates", File.dirname(__FILE__))
Dir.mkdir(output_dir) unless File.directory?(output_dir)
w = ChunkWriter.new(output_dir, miss_templates)
w.write_files "utf-8", '["֏","֏",{}', ',"֏","֏",{}', ',"֏","֏","֏"]', repeat2: ',"ab","ab",{}'
w.write_files "escape", '["\\"","\\"",{}', ',"\\"","\\"",{}', ',"\\"","\\"","\\""]', repeat2: ',"ab","ab",{}'
w.write_files "utf-8", '["֏","֏",{}', ',"֏","֏",{}', ',"֏","֏","֏"]', ',"ab","ab",{}'
w.write_files "escape", '["\\"","\\"",{}', ',"\\"","\\"",{}', ',"\\"","\\"","\\""]', ',"ab","ab",{}'
w.write_files "0-structurals", '"ab"', '', ''
# w.write_files "1-structurals", [ '[', '"ab"' ], [ ',', '"ab"' ], [ ',', '{', '}', ']' ]
# w.write_files "2-structurals", '["ab"', ',"ab"', [',{', '}]']

0
benchmark/genfeaturejsonseed.rb → jsonexamples/generated/genfeaturejsonseed.rb
View File

0
benchmark/miss-templates/128.txt → jsonexamples/generated/miss-templates/128.txt
View File

0
benchmark/miss-templates/64.txt → jsonexamples/generated/miss-templates/64.txt
View File