Final steps.
This commit is contained in:
Daniel Lemire
parent
444ec4ad27
commit
4582a13360
|
|
@ -7,8 +7,8 @@ project(simdjson
|
|||
|
||||
set(PROJECT_VERSION_MAJOR 0)
|
||||
set(PROJECT_VERSION_MINOR 4)
|
||||
set(PROJECT_VERSION_PATCH 0)
|
||||
set(SIMDJSON_LIB_VERSION "0.4.0" CACHE STRING "simdjson library version")
|
||||
set(PROJECT_VERSION_PATCH 1)
|
||||
set(SIMDJSON_LIB_VERSION "0.4.1" CACHE STRING "simdjson library version")
|
||||
set(SIMDJSON_LIB_SOVERSION "2" CACHE STRING "simdjson library soversion")
|
||||
set(SIMDJSON_GITHUB_REPOSITORY https://github.com/simdjson/simdjson)
|
||||
|
||||
|
|
|
|||
4
Doxyfile
4
Doxyfile
|
|
@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
|
|||
# could be handy for archiving the generated documentation or if some version
|
||||
# control system is used.
|
||||
|
||||
PROJECT_NUMBER = "0.4.0"
|
||||
PROJECT_NUMBER = "0.4.1"
|
||||
|
||||
# Using the PROJECT_BRIEF tag one can provide an optional one line description
|
||||
# for a project that appears at the top of each page and should give viewer a
|
||||
|
|
@ -1026,7 +1026,7 @@ FILTER_SOURCE_PATTERNS =
|
|||
# (index.html). This can be useful if you have a project on for instance GitHub
|
||||
# and want to reuse the introduction page also for the doxygen output.
|
||||
|
||||
USE_MDFILE_AS_MAINPAGE = doc/basics.md
|
||||
USE_MDFILE_AS_MAINPAGE = doc/basics_doxygen.md
|
||||
|
||||
#---------------------------------------------------------------------------
|
||||
# Configuration options related to source browsing
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@ available, we define the macro `SIMDJSON_HAS_STRING_VIEW`.
|
|||
When we detect that it is unavailable,
|
||||
we use [string-view-lite](https://github.com/martinmoene/string-view-lite) as a
|
||||
substitute. In such cases, we use the type alias `using string_view = nonstd::string_view;` to
|
||||
offer the same API, irrespective of the compiler and standard library. The macro
|
||||
offer the same API, irrespective of the compiler and standard library. The macro
|
||||
`SIMDJSON_HAS_STRING_VIEW` will be *undefined* to indicate that we emulate `string_view`.
|
||||
|
||||
|
||||
|
|
@ -572,12 +572,3 @@ Backwards Compatibility
|
|||
The only header file supported by simdjson is `simdjson.h`. Older versions of simdjson published a
|
||||
number of other include files such as `document.h` or `ParsedJson.h` alongside `simdjson.h`; these headers
|
||||
may be moved or removed in future versions.
|
||||
|
||||
|
||||
|
||||
Further Reading
|
||||
-------------
|
||||
|
||||
* [Performance](performance.md) shows some more advanced scenarios and how to tune for them.
|
||||
* [Implementation Selection](implementation-selection.md) describes runtime CPU detection and
|
||||
how you can work with it.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,556 @@
|
|||
The Basics
|
||||
==========
|
||||
|
||||
An overview of what you need to know to use simdjson, with examples.
|
||||
|
||||
|
||||
Requirements
|
||||
------------------
|
||||
|
||||
- A recent compiler (LLVM clang6 or better, GNU GCC 7 or better) on a 64-bit (ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
|
||||
- Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler.
|
||||
|
||||
Including simdjson
|
||||
------------------
|
||||
|
||||
To include simdjson, copy the simdjson.h and simdjson.cpp files from the singleheader directory
|
||||
into your project. Then include the header file in your project with:
|
||||
|
||||
```
|
||||
#include "simdjson.h"
|
||||
using namespace simdjson; // optional
|
||||
```
|
||||
|
||||
You can compile with:
|
||||
|
||||
```
|
||||
c++ myproject.cpp simdjson.cpp
|
||||
```
|
||||
|
||||
Note:
|
||||
- Users on macOS and other platforms were default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ myproject.cpp simdjson.cpp`).
|
||||
- Visual Studio users should compile with the `_CRT_SECURE_NO_WARNINGS` flag to avoid warnings with respect to our use of standard C functions such as `fopen`.
|
||||
|
||||
Using simdjson as a CMake dependency
|
||||
------------------
|
||||
|
||||
You can include the simdjson repository as a folder in your CMake project. In the parent
|
||||
`CMakeLists.txt`, include the following lines:
|
||||
|
||||
```
|
||||
set(SIMDJSON_JUST_LIBRARY ON CACHE STRING "Build just the library, nothing else." FORCE)
|
||||
add_subdirectory(simdjson EXCLUDE_FROM_ALL)
|
||||
```
|
||||
|
||||
Elsewhere in your project, you can declare dependencies on simdjson with lines such as these:
|
||||
|
||||
```
|
||||
add_executable(myprogram myprogram.cpp)
|
||||
target_link_libraries(myprogram simdjson)
|
||||
```
|
||||
|
||||
See [our CMake demonstration](https://github.com/simdjson/cmakedemo).
|
||||
|
||||
The Basics: Loading and Parsing JSON Documents
|
||||
----------------------------------------------
|
||||
|
||||
The simdjson library offers a simple DOM tree API, which you can access by creating a
|
||||
`dom::parser` and calling the `load()` method:
|
||||
|
||||
```
|
||||
dom::parser parser;
|
||||
dom::element doc = parser.load(filename); // load and parse a file
|
||||
```
|
||||
|
||||
Or by creating a padded string (for efficiency reasons, simdjson requires a string with
|
||||
SIMDJSON_PADDING bytes at the end) and calling `parse()`:
|
||||
|
||||
```
|
||||
dom::parser parser;
|
||||
dom::element doc = parser.parse("[1,2,3]"_padded); // parse a string
|
||||
```
|
||||
|
||||
The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance.
|
||||
|
||||
During the`load` or `parse` calls, neither the input file nor the input string are ever modified. After calling `load` or `parse`, the source (either a file or a string) can be safely discarded. All of the JSON data is stored in the `parser` instance. The parsed document is also immutable in simdjson: you do not modify it by accessing it.
|
||||
|
||||
For best performance, a `parser` instance should be reused over several files: otherwise you will needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson.
|
||||
|
||||
|
||||
Using the Parsed JSON
|
||||
---------------------
|
||||
|
||||
Once you have an element, you can navigate it with idiomatic C++ iterators, operators and casts.
|
||||
|
||||
* **Extracting Values (with exceptions):** You can cast a JSON element to a native type: `double(element)` or
|
||||
`double x = json_element`. This works for double, uint64_t, int64_t, bool,
|
||||
dom::object and dom::array. An exception is thrown if the cast is not possible.
|
||||
* **Extracting Values (without expceptions):** You can use a variant usage of `get()` with error codes to avoid exceptions. You first declare the variable of the appropriate type (`double`, `uint64_t`, `int64_t`, `bool`,
|
||||
`dom::object` and `dom::array`) and pass it by reference to `get()` which gives you back an error code: e.g.,
|
||||
```
|
||||
simdjson::error_code error;
|
||||
simdjson::padded_string numberstring = "1.2"_padded; // our JSON input ("1.2")
|
||||
simdjson::dom::parser parser;
|
||||
double value; // variable where we store the value to be parsed
|
||||
error = parser.parse(numberstring).get(value);
|
||||
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
|
||||
std::cout << "I parsed " << value << " from " << numberstring.data() << std::endl;
|
||||
```
|
||||
* **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`.
|
||||
* **Array Iteration:** To iterate through an array, use `for (auto value : array) { ... }`. If you
|
||||
know the type of the value, you can cast it right there, too! `for (double value : array) { ... }`
|
||||
* **Object Iteration:** You can iterate through an object's fields, too: `for (auto [key, value] : object)`
|
||||
* **Array Index:** To get at an array value by index, use the at() method: `array.at(0)` gets the
|
||||
first element.
|
||||
> Note that array[0] does not compile, because implementing [] gives the impression indexing is a
|
||||
> O(1) operation, which it is not presently in simdjson. Instead, you should iterate over the elements
|
||||
> using a for-loop, as in our examples.
|
||||
* **Array and Object size** Given an array or an object, you can get its size (number of elements or keys)
|
||||
with the `size()` method.
|
||||
* **Checking an Element Type:** You can check an element's type with `element.type()`. It
|
||||
returns an `element_type`.
|
||||
|
||||
|
||||
Here are some examples of all of the above:
|
||||
|
||||
```
|
||||
auto cars_json = R"( [
|
||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||
] )"_padded;
|
||||
dom::parser parser;
|
||||
|
||||
// Iterating through an array of objects
|
||||
for (dom::object car : parser.parse(cars_json)) {
|
||||
// Accessing a field by name
|
||||
cout << "Make/Model: " << car["make"] << "/" << car["model"] << endl;
|
||||
|
||||
// Casting a JSON element to an integer
|
||||
uint64_t year = car["year"];
|
||||
cout << "- This car is " << 2020 - year << "years old." << endl;
|
||||
|
||||
// Iterating through an array of floats
|
||||
double total_tire_pressure = 0;
|
||||
for (double tire_pressure : car["tire_pressure"]) {
|
||||
total_tire_pressure += tire_pressure;
|
||||
}
|
||||
cout << "- Average tire pressure: " << (total_tire_pressure / 4) << endl;
|
||||
|
||||
// Writing out all the information about the car
|
||||
for (auto field : car) {
|
||||
cout << "- " << field.key << ": " << field.value << endl;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Here is a different example illustrating the same ideas:
|
||||
|
||||
```
|
||||
auto abstract_json = R"( [
|
||||
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
||||
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
||||
] )"_padded;
|
||||
dom::parser parser;
|
||||
|
||||
// Parse and iterate through an array of objects
|
||||
for (dom::object obj : parser.parse(abstract_json)) {
|
||||
for(const auto& key_value : obj) {
|
||||
cout << "key: " << key_value.key << " : ";
|
||||
dom::object innerobj = key_value.value;
|
||||
cout << "a: " << double(innerobj["a"]) << ", ";
|
||||
cout << "b: " << double(innerobj["b"]) << ", ";
|
||||
cout << "c: " << int64_t(innerobj["c"]) << endl;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
And another one:
|
||||
|
||||
|
||||
```
|
||||
auto abstract_json = R"(
|
||||
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
|
||||
dom::parser parser;
|
||||
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
|
||||
cout << "number: " << v << endl;
|
||||
```
|
||||
|
||||
|
||||
C++11 Support and string_view
|
||||
-------------
|
||||
|
||||
The simdjson library builds on compilers supporting the [C++11 standard](https://en.wikipedia.org/wiki/C%2B%2B11). It is also a strict requirement: we have no plan to support older C++ compilers.
|
||||
|
||||
We represent parsed strings in simdjson using the `std::string_view` class. It avoids
|
||||
the need to copy the data, as would be necessary with the `std::string` class. It also
|
||||
avoids the pitfalls of null-terminated C strings.
|
||||
|
||||
The `std::string_view` class has become standard as part of C++17 but it is not always available
|
||||
on compilers which only supports C++11. When we detect that `string_view` is natively
|
||||
available, we define the macro `SIMDJSON_HAS_STRING_VIEW`.
|
||||
|
||||
When we detect that it is unavailable,
|
||||
we use [string-view-lite](https://github.com/martinmoene/string-view-lite) as a
|
||||
substitute. In such cases, we use the type alias `using string_view = nonstd::string_view;` to
|
||||
offer the same API, irrespective of the compiler and standard library. The macro
|
||||
`SIMDJSON_HAS_STRING_VIEW` will be *undefined* to indicate that we emulate `string_view`.
|
||||
|
||||
|
||||
C++17 Support
|
||||
-------------
|
||||
|
||||
While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
|
||||
|
||||
```
|
||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
||||
dom::parser parser;
|
||||
dom::object object;
|
||||
auto error = parser.parse(json).get(object);
|
||||
if (error) { cerr << error << endl; return; }
|
||||
for (auto [key, value] : object) {
|
||||
cout << key << " = " << value << endl;
|
||||
}
|
||||
```
|
||||
|
||||
For comparison, here is the C++ 11 version of the same code:
|
||||
|
||||
```
|
||||
// C++ 11 version for comparison
|
||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
||||
dom::parser parser;
|
||||
dom::object object;
|
||||
auto error = parser.parse(json).get(object);
|
||||
if (!error) { cerr << error << endl; return; }
|
||||
for (dom::key_value_pair field : object) {
|
||||
cout << field.key << " = " << field.value << endl;
|
||||
}
|
||||
```
|
||||
|
||||
Minifying JSON strings without parsing
|
||||
----------------------
|
||||
|
||||
In some cases, you may have valid JSON strings that you do not wish to parse but that you wish to minify. That is, you wish to remove all unnecessary spaces. We have a fast function for this purpose (`minify`). This function does not validate your content, and it does not parse it. Instead, it assumes that your string is valid UTF-8. It is much faster than parsing the string and re-serializing it in minified form. Usage is relatively simple. You must pass an input pointer with a length parameter, as well as an output pointer and an output length parameter (by reference). The output length parameter is not read, but written to. The output pointer should point to a valid memory region that is slightly overallocated (by `simdjson::SIMDJSON_PADDING`) compared to the original string length. The input pointer and input length are read, but not written to.
|
||||
|
||||
```
|
||||
// Starts with a valid JSON document as a string.
|
||||
// It does not have to be null-terminated.
|
||||
const char * some_string = "[ 1, 2, 3, 4] ";
|
||||
size_t length = strlen(some_string);
|
||||
// Create a buffer to receive the minified string. Make sure that there is enough room,
|
||||
// including some padding (simdjson::SIMDJSON_PADDING).
|
||||
std::unique_ptr<char[]> buffer{new(std::nothrow) char[length + simdjson::SIMDJSON_PADDING]};
|
||||
size_t new_length{}; // It will receive the minified length.
|
||||
auto error = simdjson::minify(some_string, length, buffer.get(), new_length);
|
||||
// The buffer variable now has "[1,2,3,4]" and new_length has value 9.
|
||||
```
|
||||
|
||||
Though it does not validate the JSON input, it will detect when the document ends with an unterminated string. E.g., it would refuse to minify the string `"this string is not terminated` because of the missing final quote.
|
||||
|
||||
|
||||
UTF-8 validation (alone)
|
||||
----------------------
|
||||
|
||||
The simdjson library has fast functions to validate UTF-8 strings. They are many times faster than most functions commonly found in libraries. You can use our fast functions, even if you do not care about JSON.
|
||||
|
||||
```
|
||||
const char * some_string = "[ 1, 2, 3, 4] ";
|
||||
size_t length = strlen(some_string);
|
||||
bool is_ok = simdjson::validate_utf8(some_string, length);
|
||||
```
|
||||
|
||||
The UTF-8 validation function merely checks that the input is valid UTF-8: it works with strings in general, not just JSON strings.
|
||||
|
||||
Your input string does not need any padding. Any string will do. The `validate_utf8` function does not do any memory allocation on the heap, and it does not throw exceptions.
|
||||
|
||||
JSON Pointer
|
||||
------------
|
||||
|
||||
The simdjson library also supports [JSON pointer](https://tools.ietf.org/html/rfc6901) through the
|
||||
at() method, letting you reach further down into the document in a single call:
|
||||
|
||||
```
|
||||
auto cars_json = R"( [
|
||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||
] )"_padded;
|
||||
dom::parser parser;
|
||||
dom::element cars = parser.parse(cars_json);
|
||||
cout << cars.at("0/tire_pressure/1") << endl; // Prints 39.9
|
||||
```
|
||||
|
||||
Error Handling
|
||||
--------------
|
||||
|
||||
All simdjson APIs that can fail return `simdjson_result<T>`, which is a <value, error_code>
|
||||
pair. You can retrieve the value with .get(), like so:
|
||||
|
||||
```
|
||||
dom::element doc;
|
||||
auto error = parser.parse(json).get(doc);
|
||||
if (error) { cerr << error << endl; exit(1); }
|
||||
```
|
||||
|
||||
When you use the code this way, it is your responsibility to check for error before using the
|
||||
result: if there is an error, the result value will not be valid and using it will caused undefined
|
||||
behavior.
|
||||
|
||||
We can write a "quick start" example where we attempt to parse a file and access some data, without triggering exceptions:
|
||||
|
||||
```
|
||||
#include "simdjson.h"
|
||||
|
||||
int main(void) {
|
||||
simdjson::dom::parser parser;
|
||||
|
||||
simdjson::dom::element tweets;
|
||||
auto error = parser.load("twitter.json").get(tweets);
|
||||
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
|
||||
|
||||
simdjson::dom::element res;
|
||||
if ((error = tweets["search_metadata"]["count"].get(res))) {
|
||||
std::cerr << "could not access keys" << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
std::cout << res << " results." << std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
### Error Handling Example
|
||||
|
||||
This is how the example in "Using the Parsed JSON" could be written using only error code checking:
|
||||
|
||||
```
|
||||
auto cars_json = R"( [
|
||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||
] )"_padded;
|
||||
dom::parser parser;
|
||||
dom::array cars;
|
||||
auto error = parser.parse(cars_json).get(cars);
|
||||
if (error) { cerr << error << endl; exit(1); }
|
||||
|
||||
// Iterating through an array of objects
|
||||
for (dom::element car_element : cars) {
|
||||
dom::object car;
|
||||
if ((error = car_element.get(car))) { cerr << error << endl; exit(1); }
|
||||
|
||||
// Accessing a field by name
|
||||
std::string_view make, model;
|
||||
if ((error = car["make"].get(make))) { cerr << error << endl; exit(1); }
|
||||
if ((error = car["model"].get(model))) { cerr << error << endl; exit(1); }
|
||||
cout << "Make/Model: " << make << "/" << model << endl;
|
||||
|
||||
// Casting a JSON element to an integer
|
||||
uint64_t year;
|
||||
if ((error = car["year"].get(year))) { cerr << error << endl; exit(1); }
|
||||
cout << "- This car is " << 2020 - year << "years old." << endl;
|
||||
|
||||
// Iterating through an array of floats
|
||||
double total_tire_pressure = 0;
|
||||
dom::array tire_pressure_array;
|
||||
if ((error = car["tire_pressure"].get(tire_pressure_array))) { cerr << error << endl; exit(1); }
|
||||
for (dom::element tire_pressure_element : tire_pressure_array) {
|
||||
double tire_pressure;
|
||||
if ((error = tire_pressure_element.get(tire_pressure))) { cerr << error << endl; exit(1); }
|
||||
total_tire_pressure += tire_pressure;
|
||||
}
|
||||
cout << "- Average tire pressure: " << (total_tire_pressure / 4) << endl;
|
||||
|
||||
// Writing out all the information about the car
|
||||
for (auto field : car) {
|
||||
cout << "- " << field.key << ": " << field.value << endl;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Here is another example:
|
||||
|
||||
```
|
||||
auto abstract_json = R"( [
|
||||
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
||||
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
||||
] )"_padded;
|
||||
dom::parser parser;
|
||||
dom::array array;
|
||||
auto error = parser.parse(abstract_json).get(array);
|
||||
if (error) { cerr << error << endl; exit(1); }
|
||||
// Iterate through an array of objects
|
||||
for (dom::element elem : array) {
|
||||
dom::object obj;
|
||||
if ((error = elem.get(obj))) { cerr << error << endl; exit(1); }
|
||||
for (auto & key_value : obj) {
|
||||
cout << "key: " << key_value.key << " : ";
|
||||
dom::object innerobj;
|
||||
if ((error = key_value.value.get(innerobj))) { cerr << error << endl; exit(1); }
|
||||
|
||||
double va, vb;
|
||||
if ((error = innerobj["a"].get(va))) { cerr << error << endl; exit(1); }
|
||||
cout << "a: " << va << ", ";
|
||||
if ((error = innerobj["b"].get(vc))) { cerr << error << endl; exit(1); }
|
||||
cout << "b: " << vb << ", ";
|
||||
|
||||
int64_t vc;
|
||||
if ((error = innerobj["c"].get(vc))) { cerr << error << endl; exit(1); }
|
||||
cout << "c: " << vc << endl;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
And another one:
|
||||
|
||||
```
|
||||
auto abstract_json = R"(
|
||||
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
|
||||
dom::parser parser;
|
||||
double v;
|
||||
auto error = parser.parse(abstract_json)["str"]["123"]["abc"].get(v);
|
||||
if (error) { cerr << error << endl; exit(1); }
|
||||
cout << "number: " << v << endl;
|
||||
```
|
||||
|
||||
Notice how we can string several operations (`parser.parse(abstract_json)["str"]["123"]["abc"].get(v)`) and only check for the error once, a strategy we call *error chaining*.
|
||||
|
||||
The next two functions will take as input a JSON document containing an array with a single element, either a string or a number. They return true upon success.
|
||||
|
||||
```
|
||||
simdjson::dom::parser parser{};
|
||||
|
||||
bool parse_double(const char *j, double &d) {
|
||||
auto error = parser.parse(j, std::strlen(j))
|
||||
.at(0)
|
||||
.get(d, error);
|
||||
if (error) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
bool parse_string(const char *j, std::string &s) {
|
||||
std::string_view answer;
|
||||
auto error = parser.parse(j,strlen(j))
|
||||
.at(0)
|
||||
.get(answer, error);
|
||||
if (error) { return false; }
|
||||
s.assign(answer.data(), answer.size());
|
||||
return true;
|
||||
}
|
||||
```
|
||||
|
||||
### Exceptions
|
||||
|
||||
Users more comfortable with an exception flow may choose to directly cast the `simdjson_result<T>` to the desired type:
|
||||
|
||||
```
|
||||
dom::element doc = parser.parse(json); // Throws an exception if there was an error!
|
||||
```
|
||||
|
||||
When used this way, a `simdjson_error` exception will be thrown if an error occurs, preventing the
|
||||
program from continuing if there was an error.
|
||||
|
||||
Tree Walking and JSON Element Types
|
||||
-----------------------------------
|
||||
|
||||
Sometimes you don't necessarily have a document with a known type, and are trying to generically
|
||||
inspect or walk over JSON elements. To do that, you can use iterators and the type() method. For
|
||||
example, here's a quick and dirty recursive function that verbosely prints the JSON document as JSON
|
||||
(* ignoring nuances like trailing commas and escaping strings, for brevity's sake):
|
||||
|
||||
```
|
||||
void print_json(dom::element element) {
|
||||
switch (element.type()) {
|
||||
case dom::element_type::ARRAY:
|
||||
cout << "[";
|
||||
for (dom::element child : dom::array(element)) {
|
||||
print_json(child);
|
||||
cout << ",";
|
||||
}
|
||||
cout << "]";
|
||||
break;
|
||||
case dom::element_type::OBJECT:
|
||||
cout << "{";
|
||||
for (dom::key_value_pair field : dom::object(element)) {
|
||||
cout << "\"" << field.key << "\": ";
|
||||
print_json(field.value);
|
||||
}
|
||||
cout << "}";
|
||||
break;
|
||||
case dom::element_type::INT64:
|
||||
cout << int64_t(element) << endl;
|
||||
break;
|
||||
case dom::element_type::UINT64:
|
||||
cout << uint64_t(element) << endl;
|
||||
break;
|
||||
case dom::element_type::DOUBLE:
|
||||
cout << double(element) << endl;
|
||||
break;
|
||||
case dom::element_type::STRING:
|
||||
cout << std::string_view(element) << endl;
|
||||
break;
|
||||
case dom::element_type::BOOL:
|
||||
cout << bool(element) << endl;
|
||||
break;
|
||||
case dom::element_type::NULL_VALUE:
|
||||
cout << "null" << endl;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void basics_treewalk_1() {
|
||||
dom::parser parser;
|
||||
print_json(parser.load("twitter.json"));
|
||||
}
|
||||
```
|
||||
|
||||
Newline-Delimited JSON (ndjson) and JSON lines
|
||||
----------------------------------------------
|
||||
|
||||
The simdjson library also support multithreaded JSON streaming through a large file containing many
|
||||
smaller JSON documents in either [ndjson](http://ndjson.org) or [JSON lines](http://jsonlines.org)
|
||||
format. If your JSON documents all contain arrays or objects, we even support direct file
|
||||
concatenation without whitespace. The concatenated file has no size restrictions (including larger
|
||||
than 4GB), though each individual document must be no larger than 4 GB.
|
||||
|
||||
Here is a simple example, given "x.json" with this content:
|
||||
|
||||
```json
|
||||
{ "foo": 1 }
|
||||
{ "foo": 2 }
|
||||
{ "foo": 3 }
|
||||
```
|
||||
|
||||
```
|
||||
dom::parser parser;
|
||||
dom::document_stream docs = parser.load_many(filename);
|
||||
for (dom::element doc : docs) {
|
||||
cout << doc["foo"] << endl;
|
||||
}
|
||||
// Prints 1 2 3
|
||||
```
|
||||
|
||||
In-memory ndjson strings can be parsed as well, with `parser.parse_many(string)`.
|
||||
|
||||
Both `load_many` and `parse_many` take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents.
|
||||
|
||||
|
||||
Thread Safety
|
||||
-------------
|
||||
|
||||
We built simdjson with thread safety in mind.
|
||||
|
||||
The simdjson library is single-threaded except for `parse_many` which may use secondary threads under its control when the library is compiled with thread support.
|
||||
|
||||
|
||||
We recommend using one `dom::parser` object per thread in which case the library is thread-safe.
|
||||
It is unsafe to reuse a `dom::parser` object between different threads.
|
||||
The parsed results (`dom::document`, `dom::element`, `array`, `object`) depend on the `dom::parser`, etc. therefore it is also potentially unsafe to use the result of the parsing between different threads.
|
||||
|
||||
The CPU detection, which runs the first time parsing is attempted and switches to the fastest
|
||||
parser for your CPU, is transparent and thread-safe.
|
||||
|
||||
Backwards Compatibility
|
||||
-----------------------
|
||||
|
||||
The only header file supported by simdjson is `simdjson.h`. Older versions of simdjson published a
|
||||
number of other include files such as `document.h` or `ParsedJson.h` alongside `simdjson.h`; these headers
|
||||
may be moved or removed in future versions.
|
||||
|
|
@ -81,11 +81,3 @@ can select the CPU architecture yourself:
|
|||
// Use the fallback implementation, even though my machine is fast enough for anything
|
||||
simdjson::active_implementation = simdjson::available_implementations["fallback"];
|
||||
```
|
||||
|
||||
|
||||
Further Reading
|
||||
-------------
|
||||
|
||||
* [Performance](performance.md) shows some more advanced scenarios and how to tune for them.
|
||||
* [Implementation Selection](implementation-selection.md) describes runtime CPU detection and
|
||||
how you can work with it.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
parse_many
|
||||
==========
|
||||
|
||||
An interface providing features to work with files or streams containing multiple JSON documents.
|
||||
An interface providing features to work with files or streams containing multiple JSON documents.
|
||||
As fast and convenient as possible.
|
||||
|
||||
Contents
|
||||
|
|
@ -26,7 +26,7 @@ values as an array, or a possibly indeterminate-length or never-
|
|||
ending sequence of values, JSON becomes difficult to work with.
|
||||
|
||||
Consider a sequence of one million values, each possibly one kilobyte
|
||||
when encoded -- roughly one gigabyte. It is often desirable to process such a dataset incrementally
|
||||
when encoded -- roughly one gigabyte. It is often desirable to process such a dataset incrementally
|
||||
without having to first read all of it before beginning to produce results.
|
||||
|
||||
Performance
|
||||
|
|
@ -109,7 +109,7 @@ format, we support any file that contains any amount of valid JSON document, **s
|
|||
or more character that is considered whitespace** by the JSON spec. Anything that is
|
||||
not whitespace will be parsed as a JSON document and could lead to failure.
|
||||
|
||||
Whitespace Characters:
|
||||
Whitespace Characters:
|
||||
- **Space**
|
||||
- **Linefeed**
|
||||
- **Carriage return**
|
||||
|
|
@ -137,16 +137,16 @@ From [jsonlines.org](http://jsonlines.org/examples/):
|
|||
["Gilbert", "2013", 24, true]
|
||||
["Alexa", "2013", 29, true]
|
||||
["May", "2012B", 14, false]
|
||||
["Deloise", "2012A", 19, true]
|
||||
["Deloise", "2012A", 19, true]
|
||||
```
|
||||
CSV seems so easy that many programmers have written code to generate it themselves, and almost every implementation is
|
||||
different. Handling broken CSV files is a common and frustrating task. CSV has no standard encoding, no standard column
|
||||
CSV seems so easy that many programmers have written code to generate it themselves, and almost every implementation is
|
||||
different. Handling broken CSV files is a common and frustrating task. CSV has no standard encoding, no standard column
|
||||
separator and multiple character escaping standards. String is the only type supported for cell values, so some programs
|
||||
attempt to guess the correct types.
|
||||
|
||||
|
||||
JSON Lines handles tabular data cleanly and without ambiguity. Cells may use the standard JSON types.
|
||||
|
||||
The biggest missing piece is an import/export filter for popular spreadsheet programs so that non-programmers can use
|
||||
|
||||
The biggest missing piece is an import/export filter for popular spreadsheet programs so that non-programmers can use
|
||||
this format.
|
||||
|
||||
- **Easy Nested Data**
|
||||
|
|
@ -156,13 +156,5 @@ From [jsonlines.org](http://jsonlines.org/examples/):
|
|||
{"name": "May", "wins": []}
|
||||
{"name": "Deloise", "wins": [["three of a kind", "5♣"]]}
|
||||
```
|
||||
JSON Lines' biggest strength is in handling lots of similar nested data structures. One .jsonl file is easier to
|
||||
JSON Lines' biggest strength is in handling lots of similar nested data structures. One .jsonl file is easier to
|
||||
work with than a directory full of XML files.
|
||||
|
||||
|
||||
Further Reading
|
||||
-------------
|
||||
|
||||
* [Performance](performance.md) shows some more advanced scenarios and how to tune for them.
|
||||
* [Implementation Selection](implementation-selection.md) describes runtime CPU detection and
|
||||
how you can work with it.
|
||||
|
|
|
|||
|
|
@ -170,10 +170,3 @@ On some Intel processors, using SIMD instructions in a sustained manner on the s
|
|||
- Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).
|
||||
|
||||
The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. Thus there should be no downclocking due to simdjson on recent processors. You may still be worried about which SIMD instruction set is used by simdjson. Thankfully, [you can always determine and change which architecture-specific implementation is used](implementation-selection.md). Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.
|
||||
|
||||
|
||||
Further Reading
|
||||
-------------
|
||||
|
||||
* [Implementation Selection](implementation-selection.md) describes runtime CPU detection and
|
||||
how you can work with it.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
#define SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
||||
/** The version of simdjson being used (major.minor.revision) */
|
||||
#define SIMDJSON_VERSION 0.4.0
|
||||
#define SIMDJSON_VERSION 0.4.1
|
||||
|
||||
namespace simdjson {
|
||||
enum {
|
||||
|
|
@ -19,7 +19,7 @@ enum {
|
|||
/**
|
||||
* The revision (major.minor.REVISION) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_REVISION = 0
|
||||
SIMDJSON_VERSION_REVISION = 1
|
||||
};
|
||||
} // namespace simdjson
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* auto-generated on Fri Jun 26 15:35:58 UTC 2020. Do not edit! */
|
||||
/* auto-generated on Fri 26 Jun 2020 20:03:28 EDT. Do not edit! */
|
||||
|
||||
#include <iostream>
|
||||
#include "simdjson.h"
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* auto-generated on Fri Jun 26 15:35:58 UTC 2020. Do not edit! */
|
||||
/* auto-generated on Fri 26 Jun 2020 20:03:28 EDT. Do not edit! */
|
||||
/* begin file src/simdjson.cpp */
|
||||
#include "simdjson.h"
|
||||
|
||||
|
|
@ -213,6 +213,10 @@ static inline uint32_t detect_supported_architectures() {
|
|||
/* begin file src/simdprune_tables.h */
|
||||
#ifndef SIMDJSON_SIMDPRUNE_TABLES_H
|
||||
#define SIMDJSON_SIMDPRUNE_TABLES_H
|
||||
|
||||
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace simdjson { // table modified and copied from
|
||||
|
|
@ -340,6 +344,8 @@ static const uint64_t thintable_epi8[256] = {
|
|||
|
||||
} // namespace simdjson
|
||||
|
||||
|
||||
#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
#endif // SIMDJSON_SIMDPRUNE_TABLES_H
|
||||
/* end file src/simdprune_tables.h */
|
||||
|
||||
|
|
@ -4037,7 +4043,15 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
|
|||
// It was described in
|
||||
// Clinger WD. How to read floating point numbers accurately.
|
||||
// ACM SIGPLAN Notices. 1990
|
||||
#ifndef FLT_EVAL_METHOD
|
||||
#error "FLT_EVAL_METHOD should be defined, please include cfloat."
|
||||
#endif
|
||||
#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
|
||||
// We cannot be certain that x/y is rounded to nearest.
|
||||
if (0 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#else
|
||||
if (-22 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#endif
|
||||
// convert the integer into a double. This is lossless since
|
||||
// 0 <= i <= 2^53 - 1.
|
||||
double d = double(i);
|
||||
|
|
@ -6194,7 +6208,15 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
|
|||
// It was described in
|
||||
// Clinger WD. How to read floating point numbers accurately.
|
||||
// ACM SIGPLAN Notices. 1990
|
||||
#ifndef FLT_EVAL_METHOD
|
||||
#error "FLT_EVAL_METHOD should be defined, please include cfloat."
|
||||
#endif
|
||||
#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
|
||||
// We cannot be certain that x/y is rounded to nearest.
|
||||
if (0 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#else
|
||||
if (-22 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#endif
|
||||
// convert the integer into a double. This is lossless since
|
||||
// 0 <= i <= 2^53 - 1.
|
||||
double d = double(i);
|
||||
|
|
@ -9498,7 +9520,15 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
|
|||
// It was described in
|
||||
// Clinger WD. How to read floating point numbers accurately.
|
||||
// ACM SIGPLAN Notices. 1990
|
||||
#ifndef FLT_EVAL_METHOD
|
||||
#error "FLT_EVAL_METHOD should be defined, please include cfloat."
|
||||
#endif
|
||||
#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
|
||||
// We cannot be certain that x/y is rounded to nearest.
|
||||
if (0 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#else
|
||||
if (-22 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#endif
|
||||
// convert the integer into a double. This is lossless since
|
||||
// 0 <= i <= 2^53 - 1.
|
||||
double d = double(i);
|
||||
|
|
@ -12776,7 +12806,15 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
|
|||
// It was described in
|
||||
// Clinger WD. How to read floating point numbers accurately.
|
||||
// ACM SIGPLAN Notices. 1990
|
||||
#ifndef FLT_EVAL_METHOD
|
||||
#error "FLT_EVAL_METHOD should be defined, please include cfloat."
|
||||
#endif
|
||||
#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
|
||||
// We cannot be certain that x/y is rounded to nearest.
|
||||
if (0 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#else
|
||||
if (-22 <= power && power <= 22 && i <= 9007199254740991) {
|
||||
#endif
|
||||
// convert the integer into a double. This is lossless since
|
||||
// 0 <= i <= 2^53 - 1.
|
||||
double d = double(i);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* auto-generated on Fri Jun 26 15:35:58 UTC 2020. Do not edit! */
|
||||
/* auto-generated on Fri 26 Jun 2020 20:03:28 EDT. Do not edit! */
|
||||
/* begin file include/simdjson.h */
|
||||
#ifndef SIMDJSON_H
|
||||
#define SIMDJSON_H
|
||||
|
|
@ -58,6 +58,7 @@
|
|||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cfloat>
|
||||
|
||||
|
||||
#ifdef _MSC_VER
|
||||
|
|
@ -95,20 +96,23 @@
|
|||
#define SIMDJSON_IS_ARM64 1
|
||||
#else
|
||||
#define SIMDJSON_IS_32BITS 1
|
||||
|
||||
// We do not support 32-bit platforms, but it can be
|
||||
// handy to identify them.
|
||||
#if defined(_M_IX86) || defined(__i386__)
|
||||
#define SIMDJSON_IS_X86_32BITS 1
|
||||
#elif defined(__arm__) || defined(_M_ARM)
|
||||
#define SIMDJSON_IS_ARM_32BITS 1
|
||||
#endif
|
||||
|
||||
#endif // defined(__x86_64__) || defined(_M_AMD64)
|
||||
|
||||
#ifdef SIMDJSON_IS_32BITS
|
||||
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(__GNUC__)
|
||||
#pragma message("The simdjson library is designed\
|
||||
for 64-bit processors and it seems that you are not \
|
||||
#pragma message("The simdjson library is designed \
|
||||
for 64-bit processors and it seems that you are not \
|
||||
compiling for a known 64-bit platform. All fast kernels \
|
||||
will be disabled and performance may be poor. Please \
|
||||
use a 64-bit target such as x64 or 64-bit ARM.")
|
||||
#else
|
||||
#error "The simdjson library is designed\
|
||||
for 64-bit processors. It seems that you are not \
|
||||
compiling for a known 64-bit platform."
|
||||
#endif
|
||||
#endif // SIMDJSON_IS_32BITS
|
||||
|
||||
// this is almost standard?
|
||||
|
|
@ -129,6 +133,15 @@ compiling for a known 64-bit platform."
|
|||
#define SIMDJSON_IMPLEMENTATION_WESTMERE 0
|
||||
#endif // SIMDJSON_IS_ARM64
|
||||
|
||||
// Our fast kernels require 64-bit systems.
|
||||
//
|
||||
// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions.
|
||||
// Furthermore, the number of SIMD registers is reduced.
|
||||
//
|
||||
// On 32-bit ARM, we would have smaller registers.
|
||||
//
|
||||
// The simdjson users should still have the fallback kernel. It is
|
||||
// slower, but it should run everywhere.
|
||||
#if SIMDJSON_IS_X86_64
|
||||
#ifndef SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
#define SIMDJSON_IMPLEMENTATION_HASWELL 1
|
||||
|
|
@ -139,7 +152,7 @@ compiling for a known 64-bit platform."
|
|||
#define SIMDJSON_IMPLEMENTATION_ARM64 0
|
||||
#endif // SIMDJSON_IS_X86_64
|
||||
|
||||
// we are going to use runtime dispatch
|
||||
// We are going to use runtime dispatch.
|
||||
#ifdef SIMDJSON_IS_X86_64
|
||||
#ifdef __clang__
|
||||
// clang does not have GCC push pop
|
||||
|
|
@ -2019,7 +2032,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
|
|||
#define SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
||||
/** The version of simdjson being used (major.minor.revision) */
|
||||
#define SIMDJSON_VERSION 0.4.0
|
||||
#define SIMDJSON_VERSION 0.4.1
|
||||
|
||||
namespace simdjson {
|
||||
enum {
|
||||
|
|
@ -2034,7 +2047,7 @@ enum {
|
|||
/**
|
||||
* The revision (major.minor.REVISION) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_REVISION = 0
|
||||
SIMDJSON_VERSION_REVISION = 1
|
||||
};
|
||||
} // namespace simdjson
|
||||
|
||||
|
|
@ -7649,7 +7662,7 @@ really_inline T tape_ref::next_tape_value() const noexcept {
|
|||
really_inline uint32_t internal::tape_ref::get_string_length() const noexcept {
|
||||
size_t string_buf_index = size_t(tape_value());
|
||||
uint32_t len;
|
||||
memcpy(&len, &doc->string_buf[size_t(string_buf_index)], sizeof(len));
|
||||
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
|
||||
return len;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue