jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Avoiding stack allocation. (#1515)

This commit is contained in:
Daniel Lemire 2021-03-23 11:32:04 -04:00 committed by GitHub
parent 5607253be5
commit 95b4870e20
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 54 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
tests/basictests.cpp
View File

@ -124,17 +124,17 @@ namespace number_tests {
bool powers_of_two() {
std::cout << __func__ << std::endl;
char buf[1024];
std::vector<char> buf(1024);
simdjson::dom::parser parser;
for (int i = -1075; i < 1024; ++i) {// large negative values should be zero.
double expected = pow(2, i);
size_t n = snprintf(buf, sizeof(buf), "%.*e", std::numeric_limits<double>::max_digits10 - 1, expected);
if (n >= sizeof(buf)) { abort(); }
size_t n = snprintf(buf.data(), buf.size(), "%.*e", std::numeric_limits<double>::max_digits10 - 1, expected);
if (n >= buf.size()) { abort(); }
double actual;
auto error = parser.parse(buf, n).get(actual);
auto error = parser.parse(buf.data(), n).get(actual);
if (error) { std::cerr << error << std::endl; return false; }
if(actual!=expected) {
std::cerr << "JSON '" << buf << " parsed to ";
std::cerr << "JSON '" << buf.data() << " parsed to ";
fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf
SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD);
return false;
@ -218,7 +218,7 @@ namespace number_tests {
bool powers_of_ten() {
std::cout << __func__ << std::endl;
char buf[1024];
std::vector<char> buf(1024);
simdjson::dom::parser parser;
bool is_pow_correct{1e-308 == std::pow(10,-308)};
@ -227,14 +227,14 @@ namespace number_tests {
std::cout << "On your system, the pow function is busted. Sorry about that. " << std::endl;
}
for (int i = start_point; i <= 308; ++i) {// large negative values should be zero.
size_t n = snprintf(buf, sizeof(buf), "1e%d", i);
if (n >= sizeof(buf)) { abort(); }
size_t n = snprintf(buf.data(), buf.size(), "1e%d", i);
if (n >= buf.size()) { abort(); }
double actual;
auto error = parser.parse(buf, n).get(actual);
auto error = parser.parse(buf.data(), n).get(actual);
if (error) { std::cerr << error << std::endl; return false; }
double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i));
if(actual!=expected) {
std::cerr << "JSON '" << buf << " parsed to ";
std::cerr << "JSON '" << buf.data() << " parsed to ";
fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf
SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD);
return false;

40
tests/document_stream_tests.cpp
View File

@ -474,13 +474,13 @@ namespace document_stream_tests {
}
bool small_window() {
std::cout << "Running " << __func__ << std::endl;
char input[2049];
input[0] = '[';
std::vector<char> input;
input.push_back('[');
for(size_t i = 1; i < 1024; i++) {
input[2*i+1]= '1';
input[2*i+2]= i < 1023 ? ',' : ']';
input.push_back('1');
input.push_back(i < 1023 ? ',' : ']');
}
auto json = simdjson::padded_string(input,2049);
auto json = simdjson::padded_string(input.data(),input.size());
simdjson::dom::parser parser;
size_t count = 0;
size_t window_size = 1024; // deliberately too small
@ -502,13 +502,13 @@ namespace document_stream_tests {
bool window_too_small_issue1370() {
std::cout << "Running " << __func__ << std::endl;
char input[2049];
input[0] = '[';
std::vector<char> input;
input.push_back('[');
for(size_t i = 1; i < 1024; i++) {
input[2*i+1]= '1';
input[2*i+2]= i < 1023 ? ',' : ']';
input.push_back('1');
input.push_back(i < 1023 ? ',' : ']');
}
auto json = simdjson::padded_string(input,2049);
auto json = simdjson::padded_string(input.data(), input.size());
// We are going to repeat this test 1000 times so
// that if there is an issue, we are more likely to
// trigger it systematically.
@ -598,15 +598,15 @@ namespace document_stream_tests {
fflush(NULL);
const size_t n_records = 10000;
std::string data;
char buf[1024];
std::vector<char> buf(1024);
for (size_t i = 0; i < n_records; ++i) {
size_t n = snprintf(buf,
sizeof(buf),
size_t n = snprintf(buf.data(),
buf.size(),
"{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", "
"\"ete\": {\"id\": %zu, \"name\": \"eventail%zu\"}}",
i, i, (i % 2) ? "homme" : "femme", i % 10, i % 10);
if (n >= sizeof(buf)) { abort(); }
data += std::string(buf, n);
if (n >= buf.size()) { abort(); }
data += std::string(buf.data(), n);
}
for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) {
printf(".");
@ -638,15 +638,15 @@ namespace document_stream_tests {
fflush(NULL);
const size_t n_records = 10000;
std::string data;
char buf[1024];
std::vector<char> buf(1024);
for (size_t i = 0; i < n_records; ++i) {
size_t n = snprintf(buf,
sizeof(buf),
size_t n = snprintf(buf.data(),
buf.size(),
"{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", "
"\"\xC3\xA9t\xC3\xA9\": {\"id\": %zu, \"name\": \"\xC3\xA9ventail%zu\"}}",
i, i, (i % 2) ? "\xE2\xBA\x83" : "\xE2\xBA\x95", i % 10, i % 10);
if (n >= sizeof(buf)) { abort(); }
data += std::string(buf, n);
if (n >= buf.size()) { abort(); }
data += std::string(buf.data(), n);
}
for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) {
printf(".");

20
tests/document_tests.cpp
View File

@ -110,25 +110,25 @@ namespace document_tests {
std::cout << "Running " << __func__ << std::endl;
const size_t n_records = 100000;
std::vector<std::string> data;
char buf[1024];
std::vector<char> buf(1024);
for (size_t i = 0; i < n_records; ++i) {
size_t n = snprintf(buf, sizeof(buf),
size_t n = snprintf(buf.data(), buf.size(),
"{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", "
"\"school\": {\"id\": %zu, \"name\": \"school%zu\"}}",
i, i, (i % 2) ? "male" : "female", i % 10, i % 10);
if (n >= sizeof(buf)) { abort(); }
data.emplace_back(std::string(buf, n));
if (n >= buf.size()) { abort(); }
data.emplace_back(std::string(buf.data(), n));
}
for (size_t i = 0; i < n_records; ++i) {
size_t n = snprintf(buf, sizeof(buf), "{\"counter\": %f, \"array\": [%s]}", static_cast<double>(i) * 3.1416,
size_t n = snprintf(buf.data(), buf.size(), "{\"counter\": %f, \"array\": [%s]}", static_cast<double>(i) * 3.1416,
(i % 2) ? "true" : "false");
if (n >= sizeof(buf)) { abort(); }
data.emplace_back(std::string(buf, n));
if (n >= buf.size()) { abort(); }
data.emplace_back(std::string(buf.data(), n));
}
for (size_t i = 0; i < n_records; ++i) {
size_t n = snprintf(buf, sizeof(buf), "{\"number\": %e}", static_cast<double>(i) * 10000.31321321);
if (n >= sizeof(buf)) { abort(); }
data.emplace_back(std::string(buf, n));
size_t n = snprintf(buf.data(), buf.size(), "{\"number\": %e}", static_cast<double>(i) * 10000.31321321);
if (n >= buf.size()) { abort(); }
data.emplace_back(std::string(buf.data(), n));
}
data.emplace_back(std::string("true"));
data.emplace_back(std::string("false"));

3
tests/integer_tests.cpp
View File

@ -9,9 +9,6 @@
#ifndef ASSERT
#define ASSERT(x) \
{ if (!(x)) { \
char buf[4096]; \
snprintf (buf, 4096, "Failure in \"%s\", line %d\n", \
__FILE__, __LINE__); \
abort (); \
} \
}

20
tests/ondemand/ondemand_number_tests.cpp
View File

@ -28,15 +28,15 @@ namespace number_tests {
// converts the double "expected" to a padded string
auto format_into_padded=[](const double expected) -> padded_string
{
char buf[1024];
const auto n = std::snprintf(buf,
sizeof(buf),
std::vector<char> buf(1024);
const auto n = std::snprintf(buf.data(),
buf.size(),
"%.*e",
std::numeric_limits<double>::max_digits10 - 1,
expected);
const auto nz=static_cast<size_t>(n);
if (n<0 || nz >= sizeof(buf)) { std::abort(); }
return padded_string(buf, nz);
if (n<0 || nz >= buf.size()) { std::abort(); }
return padded_string(buf.data(), nz);
};
for (int i = -1075; i < 1024; ++i) {// large negative values should be zero.
@ -134,7 +134,7 @@ namespace number_tests {
bool powers_of_ten() {
std::cout << __func__ << std::endl;
char buf[1024];
std::vector<char> buf(1024);
const bool is_pow_correct{1e-308 == std::pow(10,-308)};
const int start_point = is_pow_correct ? -10000 : -307;
@ -142,14 +142,14 @@ namespace number_tests {
std::cout << "On your system, the pow function is busted. Sorry about that. " << std::endl;
}
for (int i = start_point; i <= 308; ++i) {// large negative values should be zero.
const size_t n = std::snprintf(buf, sizeof(buf), "1e%d", i);
if (n >= sizeof(buf)) { std::abort(); }
const size_t n = std::snprintf(buf.data(), buf.size(), "1e%d", i);
if (n >= buf.size()) { std::abort(); }
std::fflush(nullptr);
const double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i));
if(!test_ondemand<double>(padded_string(buf, n), [&](double actual) {
if(!test_ondemand<double>(padded_string(buf.data(), n), [&](double actual) {
if(actual!=expected) {
std::cerr << "JSON '" << buf << " parsed to ";
std::cerr << "JSON '" << buf.data() << " parsed to ";
std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf
SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD);
return false;

8
tests/random_string_number_tests.cpp
View File

@ -134,16 +134,16 @@ bool check_float(double result, const char *buf) {
* and we verify that we get the same answer.
*/
bool tester(int seed, size_t volume) {
char buffer[1024]; // large buffer (can't overflow)
std::vector<char> buffer(1024); // large buffer (can't overflow)
simdjson::dom::parser parser;
RandomEngine rand(seed);
double result;
for (size_t i = 0; i < volume; i++) {
if((i%100000) == 0) { std::cout << "."; std::cout.flush(); }
size_t length = build_random_string(rand, buffer);
auto error = parser.parse(buffer, length).get(result);
size_t length = build_random_string(rand, buffer.data());
auto error = parser.parse(buffer.data(), length).get(result);
// When we parse a (finite) number, it better match strtod.
if ((!error) && (!check_float(result, buffer))) { return false; }
if ((!error) && (!check_float(result, buffer.data()))) { return false; }
}
return true;
}