#include #include #include #include "simdjson.h" #include "test_macros.h" namespace document_tests { bool issue938() { std::vector json_strings{"[true,false]", "[1,2,3,null]", R"({"yay":"json!"})"}; simdjson::dom::parser parser1; for (simdjson::padded_string str : json_strings) { simdjson::dom::element element; ASSERT_SUCCESS( parser1.parse(str).get(element) ); std::cout << element << std::endl; } std::vector file_paths{ ADVERSARIAL_JSON, FLATADVERSARIAL_JSON, DEMO_JSON, TWITTER_TIMELINE_JSON, REPEAT_JSON, SMALLDEMO_JSON, TRUENULL_JSON}; for (auto path : file_paths) { simdjson::dom::parser parser2; simdjson::dom::element element; std::cout << "file: " << path << std::endl; ASSERT_SUCCESS( parser2.load(path).get(element) ); std::cout << element.type() << std::endl; } simdjson::dom::parser parser3; for (auto path : file_paths) { simdjson::dom::element element; std::cout << "file: " << path << std::endl; ASSERT_SUCCESS( parser3.load(path).get(element) ); std::cout << element.type() << std::endl; } return true; } // adversarial example that once triggred overruns, see https://github.com/lemire/simdjson/issues/345 bool bad_example() { std::cout << __func__ << std::endl; simdjson::padded_string badjson = "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"_padded; simdjson::dom::parser parser; ASSERT_ERROR( parser.parse(badjson), simdjson::TAPE_ERROR ); return true; } bool count_array_example() { std::cout << __func__ << std::endl; simdjson::padded_string smalljson = "[1,2,3]"_padded; simdjson::dom::parser parser; simdjson::dom::array array; ASSERT_SUCCESS( parser.parse(smalljson).get(array) ); ASSERT_EQUAL( array.size(), 3 ); return true; } bool count_object_example() { std::cout << __func__ << std::endl; simdjson::padded_string smalljson = "{\"1\":1,\"2\":1,\"3\":1}"_padded; simdjson::dom::parser parser; simdjson::dom::object object; ASSERT_SUCCESS( parser.parse(smalljson).get(object) ); ASSERT_EQUAL( object.size(), 3 ); return true; } bool padded_with_open_bracket() { std::cout << __func__ << std::endl; simdjson::dom::parser parser; // This is an invalid document padded with open braces. ASSERT_ERROR( parser.parse("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false), simdjson::TAPE_ERROR); // This is a valid document padded with open braces. ASSERT_SUCCESS( parser.parse("[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false) ); return true; } // returns true if successful bool stable_test() { std::cout << __func__ << std::endl; simdjson::padded_string json = "{" "\"Image\":{" "\"Width\":800," "\"Height\":600," "\"Title\":\"View from 15th Floor\"," "\"Thumbnail\":{" "\"Url\":\"http://www.example.com/image/481989943\"," "\"Height\":125," "\"Width\":100" "}," "\"Animated\":false," "\"IDs\":[116,943.3,234,38793]" "}" "}"_padded; simdjson::dom::parser parser; std::ostringstream myStream; #if SIMDJSON_EXCEPTIONS myStream << parser.parse(json); #else simdjson::dom::element doc; simdjson_unused auto error = parser.parse(json).get(doc); myStream << doc; #endif std::string newjson = myStream.str(); if(static_cast(json) != newjson) { std::cout << "serialized json differs!" << std::endl; std::cout << static_cast(json) << std::endl; std::cout << newjson << std::endl; } return newjson == static_cast(json); } // returns true if successful bool skyprophet_test() { std::cout << "Running " << __func__ << std::endl; const size_t n_records = 100000; std::vector data; char buf[1024]; for (size_t i = 0; i < n_records; ++i) { size_t n = snprintf(buf, sizeof(buf), "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", " "\"school\": {\"id\": %zu, \"name\": \"school%zu\"}}", i, i, (i % 2) ? "male" : "female", i % 10, i % 10); if (n >= sizeof(buf)) { abort(); } data.emplace_back(std::string(buf, n)); } for (size_t i = 0; i < n_records; ++i) { size_t n = snprintf(buf, sizeof(buf), "{\"counter\": %f, \"array\": [%s]}", static_cast(i) * 3.1416, (i % 2) ? "true" : "false"); if (n >= sizeof(buf)) { abort(); } data.emplace_back(std::string(buf, n)); } for (size_t i = 0; i < n_records; ++i) { size_t n = snprintf(buf, sizeof(buf), "{\"number\": %e}", static_cast(i) * 10000.31321321); if (n >= sizeof(buf)) { abort(); } data.emplace_back(std::string(buf, n)); } data.emplace_back(std::string("true")); data.emplace_back(std::string("false")); data.emplace_back(std::string("null")); data.emplace_back(std::string("0.1")); size_t maxsize = 0; for (auto &s : data) { if (maxsize < s.size()) maxsize = s.size(); } simdjson::dom::parser parser; size_t counter = 0; for (auto &rec : data) { if ((counter % 10000) == 0) { printf("."); fflush(NULL); } counter++; auto error = parser.parse(rec.c_str(), rec.length()).error(); if (error != simdjson::error_code::SUCCESS) { printf("Something is wrong in skyprophet_test: %s.\n", rec.c_str()); printf("Parsing failed. Error is %s\n", simdjson::error_message(error)); return false; } error = parser.parse(rec.c_str(), rec.length()).error(); if (error != simdjson::error_code::SUCCESS) { printf("Something is wrong in skyprophet_test: %s.\n", rec.c_str()); printf("Parsing failed. Error is %s\n", simdjson::error_message(error)); return false; } } printf("\n"); return true; } bool lots_of_brackets() { std::string input; for(size_t i = 0; i < 200; i++) { input += "["; } for(size_t i = 0; i < 200; i++) { input += "]"; } simdjson::dom::parser parser; auto error = parser.parse(input).error(); if (error) { std::cerr << "Error: " << simdjson::error_message(error) << std::endl; return false; } return true; } bool run() { return issue938() && padded_with_open_bracket() && bad_example() && count_array_example() && count_object_example() && stable_test() && skyprophet_test() && lots_of_brackets(); } } int main(int argc, char *argv[]) { std::cout << std::unitbuf; int c; while ((c = getopt(argc, argv, "a:")) != -1) { switch (c) { case 'a': { const simdjson::implementation *impl = simdjson::available_implementations[optarg]; if (!impl) { fprintf(stderr, "Unsupported architecture value -a %s\n", optarg); return EXIT_FAILURE; } if(!impl->supported_by_runtime_system()) { fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg); return EXIT_FAILURE; } simdjson::active_implementation = impl; break; } default: fprintf(stderr, "Unexpected argument %c\n", c); return EXIT_FAILURE; } } // this is put here deliberately to check that the documentation is correct (README), // should this fail to compile, you should update the documentation: if (simdjson::active_implementation->name() == "unsupported") { printf("unsupported CPU\n"); } // We want to know what we are testing. std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name(); std::cout << "(" << simdjson::active_implementation->description() << ")" << std::endl; std::cout << "------------------------------------------------------------" << std::endl; std::cout << "Running document tests." << std::endl; if (document_tests::run()) { std::cout << "document tests are ok." << std::endl; return EXIT_SUCCESS; } else { return EXIT_FAILURE; } }