From 95b4870e20be5f97d9dcf63b23b1c6f520c366c1 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 23 Mar 2021 11:32:04 -0400 Subject: [PATCH] Avoiding stack allocation. (#1515) --- tests/basictests.cpp | 20 ++++++------ tests/document_stream_tests.cpp | 40 ++++++++++++------------ tests/document_tests.cpp | 20 ++++++------ tests/integer_tests.cpp | 3 -- tests/ondemand/ondemand_number_tests.cpp | 20 ++++++------ tests/random_string_number_tests.cpp | 8 ++--- 6 files changed, 54 insertions(+), 57 deletions(-) diff --git a/tests/basictests.cpp b/tests/basictests.cpp index ca12f37f..8a29b42b 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -124,17 +124,17 @@ namespace number_tests { bool powers_of_two() { std::cout << __func__ << std::endl; - char buf[1024]; + std::vector buf(1024); simdjson::dom::parser parser; for (int i = -1075; i < 1024; ++i) {// large negative values should be zero. double expected = pow(2, i); - size_t n = snprintf(buf, sizeof(buf), "%.*e", std::numeric_limits::max_digits10 - 1, expected); - if (n >= sizeof(buf)) { abort(); } + size_t n = snprintf(buf.data(), buf.size(), "%.*e", std::numeric_limits::max_digits10 - 1, expected); + if (n >= buf.size()) { abort(); } double actual; - auto error = parser.parse(buf, n).get(actual); + auto error = parser.parse(buf.data(), n).get(actual); if (error) { std::cerr << error << std::endl; return false; } if(actual!=expected) { - std::cerr << "JSON '" << buf << " parsed to "; + std::cerr << "JSON '" << buf.data() << " parsed to "; fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD); return false; @@ -218,7 +218,7 @@ namespace number_tests { bool powers_of_ten() { std::cout << __func__ << std::endl; - char buf[1024]; + std::vector buf(1024); simdjson::dom::parser parser; bool is_pow_correct{1e-308 == std::pow(10,-308)}; @@ -227,14 +227,14 @@ namespace number_tests { std::cout << "On your system, the pow function is busted. Sorry about that. " << std::endl; } for (int i = start_point; i <= 308; ++i) {// large negative values should be zero. - size_t n = snprintf(buf, sizeof(buf), "1e%d", i); - if (n >= sizeof(buf)) { abort(); } + size_t n = snprintf(buf.data(), buf.size(), "1e%d", i); + if (n >= buf.size()) { abort(); } double actual; - auto error = parser.parse(buf, n).get(actual); + auto error = parser.parse(buf.data(), n).get(actual); if (error) { std::cerr << error << std::endl; return false; } double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i)); if(actual!=expected) { - std::cerr << "JSON '" << buf << " parsed to "; + std::cerr << "JSON '" << buf.data() << " parsed to "; fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD); return false; diff --git a/tests/document_stream_tests.cpp b/tests/document_stream_tests.cpp index e963af18..5fa511c2 100644 --- a/tests/document_stream_tests.cpp +++ b/tests/document_stream_tests.cpp @@ -474,13 +474,13 @@ namespace document_stream_tests { } bool small_window() { std::cout << "Running " << __func__ << std::endl; - char input[2049]; - input[0] = '['; + std::vector input; + input.push_back('['); for(size_t i = 1; i < 1024; i++) { - input[2*i+1]= '1'; - input[2*i+2]= i < 1023 ? ',' : ']'; + input.push_back('1'); + input.push_back(i < 1023 ? ',' : ']'); } - auto json = simdjson::padded_string(input,2049); + auto json = simdjson::padded_string(input.data(),input.size()); simdjson::dom::parser parser; size_t count = 0; size_t window_size = 1024; // deliberately too small @@ -502,13 +502,13 @@ namespace document_stream_tests { bool window_too_small_issue1370() { std::cout << "Running " << __func__ << std::endl; - char input[2049]; - input[0] = '['; + std::vector input; + input.push_back('['); for(size_t i = 1; i < 1024; i++) { - input[2*i+1]= '1'; - input[2*i+2]= i < 1023 ? ',' : ']'; + input.push_back('1'); + input.push_back(i < 1023 ? ',' : ']'); } - auto json = simdjson::padded_string(input,2049); + auto json = simdjson::padded_string(input.data(), input.size()); // We are going to repeat this test 1000 times so // that if there is an issue, we are more likely to // trigger it systematically. @@ -598,15 +598,15 @@ namespace document_stream_tests { fflush(NULL); const size_t n_records = 10000; std::string data; - char buf[1024]; + std::vector buf(1024); for (size_t i = 0; i < n_records; ++i) { - size_t n = snprintf(buf, - sizeof(buf), + size_t n = snprintf(buf.data(), + buf.size(), "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", " "\"ete\": {\"id\": %zu, \"name\": \"eventail%zu\"}}", i, i, (i % 2) ? "homme" : "femme", i % 10, i % 10); - if (n >= sizeof(buf)) { abort(); } - data += std::string(buf, n); + if (n >= buf.size()) { abort(); } + data += std::string(buf.data(), n); } for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) { printf("."); @@ -638,15 +638,15 @@ namespace document_stream_tests { fflush(NULL); const size_t n_records = 10000; std::string data; - char buf[1024]; + std::vector buf(1024); for (size_t i = 0; i < n_records; ++i) { - size_t n = snprintf(buf, - sizeof(buf), + size_t n = snprintf(buf.data(), + buf.size(), "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", " "\"\xC3\xA9t\xC3\xA9\": {\"id\": %zu, \"name\": \"\xC3\xA9ventail%zu\"}}", i, i, (i % 2) ? "\xE2\xBA\x83" : "\xE2\xBA\x95", i % 10, i % 10); - if (n >= sizeof(buf)) { abort(); } - data += std::string(buf, n); + if (n >= buf.size()) { abort(); } + data += std::string(buf.data(), n); } for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) { printf("."); diff --git a/tests/document_tests.cpp b/tests/document_tests.cpp index a5697ce3..f4a24077 100644 --- a/tests/document_tests.cpp +++ b/tests/document_tests.cpp @@ -110,25 +110,25 @@ namespace document_tests { std::cout << "Running " << __func__ << std::endl; const size_t n_records = 100000; std::vector data; - char buf[1024]; + std::vector buf(1024); for (size_t i = 0; i < n_records; ++i) { - size_t n = snprintf(buf, sizeof(buf), + size_t n = snprintf(buf.data(), buf.size(), "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", " "\"school\": {\"id\": %zu, \"name\": \"school%zu\"}}", i, i, (i % 2) ? "male" : "female", i % 10, i % 10); - if (n >= sizeof(buf)) { abort(); } - data.emplace_back(std::string(buf, n)); + if (n >= buf.size()) { abort(); } + data.emplace_back(std::string(buf.data(), n)); } for (size_t i = 0; i < n_records; ++i) { - size_t n = snprintf(buf, sizeof(buf), "{\"counter\": %f, \"array\": [%s]}", static_cast(i) * 3.1416, + size_t n = snprintf(buf.data(), buf.size(), "{\"counter\": %f, \"array\": [%s]}", static_cast(i) * 3.1416, (i % 2) ? "true" : "false"); - if (n >= sizeof(buf)) { abort(); } - data.emplace_back(std::string(buf, n)); + if (n >= buf.size()) { abort(); } + data.emplace_back(std::string(buf.data(), n)); } for (size_t i = 0; i < n_records; ++i) { - size_t n = snprintf(buf, sizeof(buf), "{\"number\": %e}", static_cast(i) * 10000.31321321); - if (n >= sizeof(buf)) { abort(); } - data.emplace_back(std::string(buf, n)); + size_t n = snprintf(buf.data(), buf.size(), "{\"number\": %e}", static_cast(i) * 10000.31321321); + if (n >= buf.size()) { abort(); } + data.emplace_back(std::string(buf.data(), n)); } data.emplace_back(std::string("true")); data.emplace_back(std::string("false")); diff --git a/tests/integer_tests.cpp b/tests/integer_tests.cpp index 66178b3d..977795d0 100644 --- a/tests/integer_tests.cpp +++ b/tests/integer_tests.cpp @@ -9,9 +9,6 @@ #ifndef ASSERT #define ASSERT(x) \ { if (!(x)) { \ - char buf[4096]; \ - snprintf (buf, 4096, "Failure in \"%s\", line %d\n", \ - __FILE__, __LINE__); \ abort (); \ } \ } diff --git a/tests/ondemand/ondemand_number_tests.cpp b/tests/ondemand/ondemand_number_tests.cpp index 1cabdb3f..a2caa153 100644 --- a/tests/ondemand/ondemand_number_tests.cpp +++ b/tests/ondemand/ondemand_number_tests.cpp @@ -28,15 +28,15 @@ namespace number_tests { // converts the double "expected" to a padded string auto format_into_padded=[](const double expected) -> padded_string { - char buf[1024]; - const auto n = std::snprintf(buf, - sizeof(buf), + std::vector buf(1024); + const auto n = std::snprintf(buf.data(), + buf.size(), "%.*e", std::numeric_limits::max_digits10 - 1, expected); const auto nz=static_cast(n); - if (n<0 || nz >= sizeof(buf)) { std::abort(); } - return padded_string(buf, nz); + if (n<0 || nz >= buf.size()) { std::abort(); } + return padded_string(buf.data(), nz); }; for (int i = -1075; i < 1024; ++i) {// large negative values should be zero. @@ -134,7 +134,7 @@ namespace number_tests { bool powers_of_ten() { std::cout << __func__ << std::endl; - char buf[1024]; + std::vector buf(1024); const bool is_pow_correct{1e-308 == std::pow(10,-308)}; const int start_point = is_pow_correct ? -10000 : -307; @@ -142,14 +142,14 @@ namespace number_tests { std::cout << "On your system, the pow function is busted. Sorry about that. " << std::endl; } for (int i = start_point; i <= 308; ++i) {// large negative values should be zero. - const size_t n = std::snprintf(buf, sizeof(buf), "1e%d", i); - if (n >= sizeof(buf)) { std::abort(); } + const size_t n = std::snprintf(buf.data(), buf.size(), "1e%d", i); + if (n >= buf.size()) { std::abort(); } std::fflush(nullptr); const double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i)); - if(!test_ondemand(padded_string(buf, n), [&](double actual) { + if(!test_ondemand(padded_string(buf.data(), n), [&](double actual) { if(actual!=expected) { - std::cerr << "JSON '" << buf << " parsed to "; + std::cerr << "JSON '" << buf.data() << " parsed to "; std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD); return false; diff --git a/tests/random_string_number_tests.cpp b/tests/random_string_number_tests.cpp index 127e84fd..d269f9a7 100644 --- a/tests/random_string_number_tests.cpp +++ b/tests/random_string_number_tests.cpp @@ -134,16 +134,16 @@ bool check_float(double result, const char *buf) { * and we verify that we get the same answer. */ bool tester(int seed, size_t volume) { - char buffer[1024]; // large buffer (can't overflow) + std::vector buffer(1024); // large buffer (can't overflow) simdjson::dom::parser parser; RandomEngine rand(seed); double result; for (size_t i = 0; i < volume; i++) { if((i%100000) == 0) { std::cout << "."; std::cout.flush(); } - size_t length = build_random_string(rand, buffer); - auto error = parser.parse(buffer, length).get(result); + size_t length = build_random_string(rand, buffer.data()); + auto error = parser.parse(buffer.data(), length).get(result); // When we parse a (finite) number, it better match strtod. - if ((!error) && (!check_float(result, buffer))) { return false; } + if ((!error) && (!check_float(result, buffer.data()))) { return false; } } return true; }