jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Introducing a "native" instruction set so that you do not need to do #ifdef to select the right SIMD set all the time. 

Fixing indentation.
Removing some obsolete WARN_UNUSED.
Fixing a weird warning with optind variable.
This commit is contained in:
Daniel Lemire 2019-07-01 14:18:30 -04:00
parent 1b81e7c928
commit aa78b70d69
7 changed files with 50 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
benchmark/parse.cpp
View File

@ -144,11 +144,8 @@ int main(int argc, char *argv[]) {
std::cout << "[verbose] allocated memory for parsed JSON " << std::endl; std::cout << "[verbose] allocated memory for parsed JSON " << std::endl;
} }
unified.start(); unified.start();
#ifdef __AVX2__ // The default template is simdjson::instruction_set::native.
isok = (find_structural_bits<simdjson::instruction_set::avx2>(p.data(), p.size(), pj) == simdjson::SUCCESS); isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS);
#elif defined (__ARM_NEON)
isok = (find_structural_bits<simdjson::instruction_set::neon>(p.data(), p.size(), pj) == simdjson::SUCCESS);
#endif
unified.end(results); unified.end(results);
cy1 += results[0]; cy1 += results[0];
cl1 += results[1]; cl1 += results[1];
@ -189,11 +186,8 @@ int main(int argc, char *argv[]) {
} }
auto start = std::chrono::steady_clock::now(); auto start = std::chrono::steady_clock::now();
#ifdef __AVX2__ // The default template is simdjson::instruction_set::native.
isok = (find_structural_bits<simdjson::instruction_set::avx2>(p.data(), p.size(), pj) == simdjson::SUCCESS); isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS);
#elif defined (__ARM_NEON)
isok = (find_structural_bits<simdjson::instruction_set::neon>(p.data(), p.size(), pj) == simdjson::SUCCESS);
#endif
isok = isok && (simdjson::SUCCESS == unified_machine(p.data(), p.size(), pj)); isok = isok && (simdjson::SUCCESS == unified_machine(p.data(), p.size(), pj));
auto end = std::chrono::steady_clock::now(); auto end = std::chrono::steady_clock::now();
std::chrono::duration<double> secs = end - start; std::chrono::duration<double> secs = end - start;

7
benchmark/statisticalmodel.cpp
View File

@ -180,11 +180,8 @@ int main(int argc, char *argv[]) {
results.resize(evts.size()); results.resize(evts.size());
for (uint32_t i = 0; i < iterations; i++) { for (uint32_t i = 0; i < iterations; i++) {
unified.start(); unified.start();
#ifdef __AVX2__ // The default template is simdjson::instruction_set::native.
bool isok = (find_structural_bits<simdjson::instruction_set::avx2>(p.data(), p.size(), pj) == simdjson::SUCCESS); bool isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS);
#elif defined (__ARM_NEON)
bool isok = (find_structural_bits<simdjson::instruction_set::neon>(p.data(), p.size(), pj) == simdjson::SUCCESS);
#endif
unified.end(results); unified.end(results);
cy1 += results[0]; cy1 += results[0];

40
include/simdjson/jsonparser.h
View File

@ -15,12 +15,20 @@
#include <unistd.h> #include <unistd.h>
#endif #endif
// The function that users are expected to call is json_parse.
// We have more than one such function because we want to support several
// instruction sets.
// function pointer type for json_parse // function pointer type for json_parse
using json_parse_functype = int (const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded); using json_parse_functype = int (const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded);
// Pointer that holds the json_parse implementation corresponding to the available SIMD instruction set // Pointer that holds the json_parse implementation corresponding to the available SIMD instruction set
extern json_parse_functype *json_parse_ptr; extern json_parse_functype *json_parse_ptr;
// json_parse_implementation is the generic function, it is specialized for various
// SIMD instruction sets, e.g., as json_parse_implementation<simdjson::instruction_set::avx2>
// or json_parse_implementation<simdjson::instruction_set::neon>
template<simdjson::instruction_set T> template<simdjson::instruction_set T>
int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
if (pj.bytecapacity < len) { if (pj.bytecapacity < len) {
@ -29,22 +37,22 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo
bool reallocated = false; bool reallocated = false;
if(reallocifneeded) { if(reallocifneeded) {
#ifdef ALLOW_SAME_PAGE_BUFFER_OVERRUN #ifdef ALLOW_SAME_PAGE_BUFFER_OVERRUN
// realloc is needed if the end of the memory crosses a page // realloc is needed if the end of the memory crosses a page
#ifdef _MSC_VER #ifdef _MSC_VER
SYSTEM_INFO sysInfo; SYSTEM_INFO sysInfo;
GetSystemInfo(&sysInfo); GetSystemInfo(&sysInfo);
long pagesize = sysInfo.dwPageSize; long pagesize = sysInfo.dwPageSize;
#else #else
long pagesize = sysconf (_SC_PAGESIZE); long pagesize = sysconf (_SC_PAGESIZE);
#endif #endif
////////////// //////////////
// We want to check that buf + len - 1 and buf + len - 1 + SIMDJSON_PADDING // We want to check that buf + len - 1 and buf + len - 1 + SIMDJSON_PADDING
// are in the same page. // are in the same page.
// That is, we want to check that // That is, we want to check that
// (buf + len - 1) / pagesize == (buf + len - 1 + SIMDJSON_PADDING) / pagesize // (buf + len - 1) / pagesize == (buf + len - 1 + SIMDJSON_PADDING) / pagesize
// That's true if (buf + len - 1) % pagesize + SIMDJSON_PADDING < pagesize. // That's true if (buf + len - 1) % pagesize + SIMDJSON_PADDING < pagesize.
/////////// ///////////
if ( (reinterpret_cast<uintptr_t>(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast<uintptr_t>(pagesize) ) { if ( (reinterpret_cast<uintptr_t>(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast<uintptr_t>(pagesize) ) {
#else // SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN #else // SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN
if(true) { // if not SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN, we always reallocate if(true) { // if not SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN, we always reallocate
#endif #endif
@ -53,8 +61,8 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo
if(buf == NULL) return simdjson::MEMALLOC; if(buf == NULL) return simdjson::MEMALLOC;
memcpy((void*)buf,tmpbuf,len); memcpy((void*)buf,tmpbuf,len);
reallocated = true; reallocated = true;
} } // if (true) OR if ( (reinterpret_cast<uintptr_t>(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast<uintptr_t>(pagesize) ) {
} } // if(reallocifneeded) {
int stage1_is_ok = find_structural_bits<T>(buf, len, pj); int stage1_is_ok = find_structural_bits<T>(buf, len, pj);
if(stage1_is_ok != simdjson::SUCCESS) { if(stage1_is_ok != simdjson::SUCCESS) {
pj.errorcode = stage1_is_ok; pj.errorcode = stage1_is_ok;
@ -81,7 +89,6 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo
// all bytes at and after buf + len are ignored (can be garbage). // all bytes at and after buf + len are ignored (can be garbage).
// The ParsedJson object can be reused. // The ParsedJson object can be reused.
WARN_UNUSED
inline int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { inline int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
return json_parse_ptr(buf, len, pj, reallocifneeded); return json_parse_ptr(buf, len, pj, reallocifneeded);
} }
@ -102,7 +109,6 @@ inline int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reall
// The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false, // The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
// all bytes at and after buf + len are ignored (can be garbage). // all bytes at and after buf + len are ignored (can be garbage).
// The ParsedJson object can be reused. // The ParsedJson object can be reused.
WARN_UNUSED
inline int json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { inline int json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
return json_parse_ptr(reinterpret_cast<const uint8_t *>(buf), len, pj, reallocifneeded); return json_parse_ptr(reinterpret_cast<const uint8_t *>(buf), len, pj, reallocifneeded);
} }
@ -120,7 +126,6 @@ int json_parse(const char * buf, ParsedJson &pj) = delete;
// //
// A temporary buffer is created when needed during processing // A temporary buffer is created when needed during processing
// (a copy of the input string is made). // (a copy of the input string is made).
WARN_UNUSED
inline int json_parse(const std::string &s, ParsedJson &pj) { inline int json_parse(const std::string &s, ParsedJson &pj) {
return json_parse(s.data(), s.length(), pj, true); return json_parse(s.data(), s.length(), pj, true);
} }
@ -135,7 +140,6 @@ inline int json_parse(const std::string &s, ParsedJson &pj) {
// //
// You can also check validity // You can also check validity
// by calling pj.isValid(). The same ParsedJson can be reused for other documents. // by calling pj.isValid(). The same ParsedJson can be reused for other documents.
WARN_UNUSED
inline int json_parse(const padded_string &s, ParsedJson &pj) { inline int json_parse(const padded_string &s, ParsedJson &pj) {
return json_parse(s.data(), s.length(), pj, false); return json_parse(s.data(), s.length(), pj, false);
} }

15
include/simdjson/simdjson.h
View File

@ -8,7 +8,20 @@ struct simdjson {
avx2, avx2,
sse4_2, sse4_2,
neon, neon,
none none,
// the 'native' enum class value should point at a good default on the current machine
#ifdef __AVX2__
native = avx2
#elif defined(__ARM_NEON)
native = neon
#else
// Let us assume that we have an old x64 processor, but one that has SSE (i.e., something
// that came out in the second decade of the XXIst century.
// It would be nicer to check explicitly, but there many not be a good way to do so
// that is cross-platform.
// Under Visual Studio, there is no way to check for SSE4.2 support at compile-time.
native = sse4_2
#endif
}; };
enum errorValues { enum errorValues {

4
include/simdjson/stage1_find_marks.h
View File

@ -697,7 +697,7 @@ really_inline uint64_t finalize_structurals(
return structurals; return structurals;
} }
template<simdjson::instruction_set T> template<simdjson::instruction_set T = simdjson::instruction_set::native>
WARN_UNUSED WARN_UNUSED
/*never_inline*/ int find_structural_bits(const uint8_t *buf, size_t len, /*never_inline*/ int find_structural_bits(const uint8_t *buf, size_t len,
ParsedJson &pj) { ParsedJson &pj) {
@ -848,7 +848,7 @@ WARN_UNUSED
#endif #endif
} }
template<simdjson::instruction_set T> template<simdjson::instruction_set T = simdjson::instruction_set::native>
WARN_UNUSED WARN_UNUSED
int find_structural_bits(const char *buf, size_t len, ParsedJson &pj) { int find_structural_bits(const char *buf, size_t len, ParsedJson &pj) {
return find_structural_bits<T>(reinterpret_cast<const uint8_t *>(buf), len, pj); return find_structural_bits<T>(reinterpret_cast<const uint8_t *>(buf), len, pj);

2
src/jsonparser.cpp
View File

@ -63,7 +63,7 @@ ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneede
ParsedJson pj; ParsedJson pj;
bool ok = pj.allocateCapacity(len); bool ok = pj.allocateCapacity(len);
if(ok) { if(ok) {
(void)json_parse(buf, len, pj, reallocifneeded); json_parse(buf, len, pj, reallocifneeded);
} else { } else {
std::cerr << "failure during memory allocation " << std::endl; std::cerr << "failure during memory allocation " << std::endl;
} }

10
tools/jsonstats.cpp
View File

@ -115,15 +115,15 @@ stat_t simdjson_computestats(const padded_string &p) {
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
int optind = 1; int myoptind = 1;
if (optind >= argc) { if (myoptind >= argc) {
std::cerr << "Reads json, prints stats. " << std::endl; std::cerr << "Reads json, prints stats. " << std::endl;
std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl; std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
exit(1); exit(1);
} }
const char *filename = argv[optind]; const char *filename = argv[myoptind];
if (optind + 1 < argc) { if (myoptind + 1 < argc) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; std::cerr << "warning: ignoring everything after " << argv[myoptind + 1] << std::endl;
} }
padded_string p; padded_string p;
try { try {