jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Porting to visual studio 

Now builds on Visual Studio
This commit is contained in:
Daniel Lemire 2018-12-30 21:00:19 -05:00 committed by GitHub
parent 2b3d6f6ae4
commit 58d41923fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 1585 additions and 307 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CMakeLists.txt
View File

@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 2.8)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_MACOSX_RPATH OFF)
if (NOT CMAKE_BUILD_TYPE)
message(STATUS "No build type selected, default to Release")
@ -10,8 +10,11 @@ endif()
project(simdjson)
set(SIMDJSON_LIB_NAME simdjson)
if(NOT MSVC)
option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
else()
option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library
endif()
option(SIMDJSON_BUILD_LTO "Build library with Link Time Optimization" OFF)
option(SIMDJSON_SANITIZE "Sanitize addresses" OFF)

27
README.md
View File

@ -39,15 +39,17 @@ twitter.json:
## Requirements
- Linux or macOS (currently)
- We support platforms like Linux or macOS, as well as Windows through Visual Studio 2017 or better.
- A processor with AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013, and processors from AMD starting with the Rizen)
- A recent C++ compiler (e.g., GNU GCC or LLVM CLANG), we assume C++17
- Bash (for benchmark scripts) and other common utilities (optional)
- A recent C++ compiler (e.g., GNU GCC or LLVM CLANG or Visual Studio 2017), we assume C++17
- Some benchmark scripts assume bash and other common utilities, but they are optional.
## License
This code is made available under the Apache License 2.0.
Under Windows, we build some tools using the windows/dirent_portable.h file (which is outside our library code): it under the liberal (business-friendly) MIT license.
## Code example
```C
@ -80,14 +82,14 @@ of memory allocation with each new JSON document:
const char * filename = ... //
std::string_view p = get_corpus(filename);
ParsedJson pj = build_parsed_json(p); // do the parsing
// you no longer need p at this point, can do free((void*)p.data())
// you no longer need p at this point, can do aligned_free((void*)p.data())
if( ! pj.isValid() ) {
// something went wrong
}
```
## Usage (old-school Makefile)
## Usage (old-school Makefile on platforms like Linux or macOS)
Requirements: clang or gcc and make. A system like Linux or macOS is expected.
@ -112,7 +114,7 @@ To run comparative benchmarks (with other parsers):
make benchmark
```
## Usage (CMake)
## Usage (CMake on platforms like Linux or macOS)
While in the project repository, do the following:
@ -136,6 +138,19 @@ make
make test
```
## Usage (CMake on Windows using Visual Studio)
We are assuming that you have a common Windows PC with at least Visual Studio 2017, and an x64 processor with AVX2 support (2013 Haswell or better).
- Grab the simdjosn code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/).
- Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that ``cmake`` be made available from the command line.
- Create a subdirectory within simdjson, such as ``VisualStudio``.
- Using a shell, go to this newly created directory.
- Type ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..`` in the shell while in the ``VisualStudio`` repository. (Alternatively, if you want to build a DLL, you may use the command line ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..``.)
- This last command created a Visual Studio solution file in the newly created directory (e.g., ``simdjson.sln``). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the ``Solution Explorer`` window (available from the ``View`` menu), right-click ``ALL_BUILD`` and select ``Build``. To test the code, still in the ``Solution Explorer`` window, select ``RUN_TESTS`` and select ``Build``.
## Tools
- `json2json mydoc.json` parses the document, constructs a model and then dumps back the result to standard output.

2
benchmark/benchmark.h
View File

@ -106,7 +106,7 @@ double diff(timespec start, timespec end) {
clock_gettime(CLOCK_REALTIME, &time1); \
RDTSC_START(cycles_start); \
if (test != expected) { \
printf("not expected (%d , %d )", (int)test, (int)expected); \
fprintf(stderr, "not expected (%d , %d )", (int)test, (int)expected); \
break; \
} \
RDTSC_STOP(cycles_final); \

2
benchmark/distinctuseridcompetition.cpp
View File

@ -279,5 +279,5 @@ int main(int argc, char *argv[]) {
!justdata);
BEST_TIME("sasjon ", sasjon_computestats(p).size(), size, , repeat, volume,
!justdata);
free((void*)p.data());
aligned_free((void*)p.data());
}

6
benchmark/minifiercompetition.cpp
View File

@ -137,7 +137,7 @@ int main(int argc, char *argv[]) {
ParsedJson pj;
bool isallocok = pj.allocateCapacity(p.size(), 1024);
if(!isallocok) {
printf("failed to allocate memory\n");
fprintf(stderr, "failed to allocate memory\n");
return EXIT_FAILURE;
}
BEST_TIME("simdjson orig", json_parse((const uint8_t*)buffer, p.size(), pj), true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
@ -145,12 +145,12 @@ int main(int argc, char *argv[]) {
ParsedJson pj2;
bool isallocok2 = pj2.allocateCapacity(p.size(), 1024);
if(!isallocok2) {
printf("failed to allocate memory\n");
fprintf(stderr, "failed to allocate memory\n");
return EXIT_FAILURE;
}
BEST_TIME("simdjson despaced", json_parse((const uint8_t*)buffer, minisize, pj2), true, memcpy(buffer, minibuffer, p.size()), repeat, volume, !justdata);
free((void*)p.data());
aligned_free((void*)p.data());
free(buffer);
free(ast_buffer);
free(minibuffer);

17
benchmark/parse.cpp
View File

@ -1,13 +1,17 @@
#include <assert.h>
#include <ctype.h>
#ifndef _MSC_VER
#include <unistd.h>
#include <x86intrin.h>
#include <dirent.h>
#else
#include <intrin.h>
#endif
#include <inttypes.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <x86intrin.h>
#include <algorithm>
#include <chrono>
@ -41,7 +45,7 @@ int main(int argc, char *argv[]) {
bool jsonoutput = false;
bool forceoneiteration = false;
bool justdata = false;
#ifndef _MSC_VER
int c;
while ((c = getopt(argc, argv, "1vdt")) != -1)
@ -64,6 +68,9 @@ int main(int argc, char *argv[]) {
default:
abort();
}
#else
int optind = 1;
#endif
if (optind >= argc) {
cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;
exit(1);
@ -278,9 +285,9 @@ int main(int argc, char *argv[]) {
if (dump) {
isok = isok && pj.dump_raw_tape(std::cout);
}
free((void *)p.data());
aligned_free((void *)p.data());
if (!isok) {
printf(" Parsing failed. \n ");
fprintf(stderr, " Parsing failed. \n ");
return EXIT_FAILURE;
}
return EXIT_SUCCESS;

2
benchmark/parseandstatcompetition.cpp
View File

@ -300,5 +300,5 @@ int main(int argc, char *argv[]) {
!justdata);
BEST_TIME("sasjon ", sasjon_computestats(p).valid, true, , repeat, volume,
!justdata);
free((void*)p.data());
aligned_free((void*)p.data());
}

2
benchmark/parsingcompetition.cpp
View File

@ -158,7 +158,7 @@ int main(int argc, char *argv[]) {
if(!justdata) BEST_TIME("memcpy ",
(memcpy(buffer, p.data(), p.size()) == buffer), true, , repeat,
volume, !justdata);
free((void *)p.data());
aligned_free((void *)p.data());
free(ast_buffer);
free(buffer);
}

12
benchmark/statisticalmodel.cpp
View File

@ -1,6 +1,7 @@
#include <iostream>
#ifndef _MSC_VER
#include <unistd.h>
#endif
#include "simdjson/jsonioutil.h"
#include "simdjson/jsonparser.h"
#ifdef __linux__
@ -112,14 +113,17 @@ stat_t simdjson_computestats(const std::string_view &p) {
}
int main(int argc, char *argv[]) {
int c;
while ((c = getopt(argc, argv, "")) != -1)
#ifndef _MSC_VER
int c;
while ((c = getopt(argc, argv, "")) != -1)
switch (c) {
default:
abort();
}
#else
int optind = 1;
#endif
if (optind >= argc) {
cerr << "Reads json, prints stats. " << endl;
cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;

42
include/simdjson/common_defs.h
View File

@ -1,27 +1,46 @@
#pragma once
#ifndef SIMDJSON_COMMON_DEFS_H
#define SIMDJSON_COMMON_DEFS_H
#include "simdjson/portability.h"
#include <cassert>
// the input buf should be readable up to buf + SIMDJSON_PADDING
#define SIMDJSON_PADDING sizeof(__m256i)
#define SIMDJSON_PADDING sizeof(__m256i)
#ifdef _MSC_VER
/* Microsoft C/C++-compatible compiler */
#include <intrin.h>
#else
#include <x86intrin.h>
#ifndef _MSC_VER
// Implemented using Labels as Values which works in GCC and CLANG (and maybe
// also in Intel's compiler), but won't work in MSVC.
#define SIMDJSON_USE_COMPUTED_GOTO
#endif
// Align to N-byte boundary
#define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
#define ROUNDDOWN_N(a, n) ((a) & ~((n)-1))
#define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
#ifdef _MSC_VER
#define really_inline inline
#define never_inline inline
#define UNUSED
#define WARN_UNUSED
#ifndef likely
#define likely(x) x
#endif
#ifndef unlikely
#define unlikely(x) x
#endif
#else
#define really_inline inline __attribute__((always_inline, unused))
#define never_inline inline __attribute__((noinline, unused))
@ -35,3 +54,6 @@
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
#endif // MSC_VER
#endif // COMMON_DEFS_H

6
include/simdjson/jsoncharutils.h
View File

@ -1,4 +1,5 @@
#pragma once
#ifndef SIMDJSON_JSONCHARUTILS_H
#define SIMDJSON_JSONCHARUTILS_H
#include "simdjson/common_defs.h"
#include "simdjson/parsedjson.h"
@ -96,7 +97,7 @@ uint32_t hex_to_u32_nocheck(const uint8_t *src) {
// and clz and table lookups, but JSON documents
// have few escaped code points, and the following
// function looks cheap.
//
//
// Note: we assume that surrogates are treated separately
//
inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
@ -126,3 +127,4 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
return 0; // bad r
}
#endif

11
include/simdjson/jsonformatutils.h
View File

@ -1,4 +1,5 @@
#pragma once
#ifndef SIMDJSON_JSONFORMATUTILS_H
#define SIMDJSON_JSONFORMATUTILS_H
#include <stdio.h>
#include <iostream>
@ -79,7 +80,7 @@ static inline void print_with_escapes(const unsigned char *src, std::ostream &os
default:
if (*src <= 0x1F) {
std::ios::fmtflags f(os.flags());
os << std::hex << std::setw(4) << std::setfill('0') << (int) *src;
os << std::hex << std::setw(4) << std::setfill('0') << (int) *src;
os.flags(f);
} else
os << *src;
@ -89,5 +90,7 @@ static inline void print_with_escapes(const unsigned char *src, std::ostream &os
}
static inline void print_with_escapes(const char *src, std::ostream &os) {
print_with_escapes((const unsigned char *)src, os);
}
print_with_escapes((const unsigned char *)src, os);
}
#endif

7
include/simdjson/jsonioutil.h
View File

@ -1,12 +1,11 @@
#ifndef JSONIOUTIL_H
#define JSONIOUTIL_H
#ifndef SIMDJSON_JSONIOUTIL_H
#define SIMDJSON_JSONIOUTIL_H
#include <exception>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <string_view>
#include "simdjson/common_defs.h"
@ -32,7 +31,7 @@ char * allocate_padded_buffer(size_t length);
// try {
// p = get_corpus(filename);
// } catch (const std::exception& e) {
// free((void*)p.data());
// free((void*)p.data());//use aligned_free if you plan to use VisualStudio
// std::cout << "Could not load the file " << filename << std::endl;
// }
std::string_view get_corpus(std::string filename);

9
include/simdjson/jsonminifier.h
View File

@ -1,10 +1,11 @@
#pragma once
#ifndef SIMDJSON_JSONMINIFIER_H
#define SIMDJSON_JSONMINIFIER_H
#include <cstddef>
#include <cstdint>
// Take input from buf and remove useless whitespace, write it to out; buf and
// out can be the same pointer. Result is null terminated,
// out can be the same pointer. Result is null terminated,
// return the string length (minus the null termination).
size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out);
@ -16,4 +17,6 @@ static inline size_t jsonminify(const char *buf, size_t len, char *out) {
static inline size_t jsonminify(const std::string_view & p, char *out) {
return jsonminify(p.data(), p.size(), out);
}
}
#endif

27
include/simdjson/jsonparser.h
View File

@ -1,4 +1,5 @@
#pragma once
#ifndef SIMDJSON_JSONPARSER_H
#define SIMDJSON_JSONPARSER_H
#include "simdjson/common_defs.h"
#include "simdjson/jsonioutil.h"
@ -11,7 +12,7 @@
// Parse a document found in buf, need to preallocate ParsedJson.
// Return false in case of a failure. You can also check validity
// Return false in case of a failure. You can also check validity
// by calling pj.isValid(). The same ParsedJson can be reused for other documents.
//
// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
@ -22,7 +23,7 @@ WARN_UNUSED
bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);
// Parse a document found in buf, need to preallocate ParsedJson.
// Return false in case of a failure. You can also check validity
// Return false in case of a failure. You can also check validity
// by calling pj.isValid(). The same ParsedJson can be reused for other documents.
//
// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
@ -30,12 +31,12 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne
// The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
// all bytes at and after buf + len are ignored (can be garbage).
WARN_UNUSED
static inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
return json_parse((const uint8_t *) buf, len, pj, reallocifneeded);
}
// Parse a document found in buf, need to preallocate ParsedJson.
// Return false in case of a failure. You can also check validity
// Return false in case of a failure. You can also check validity
// by calling pj.isValid(). The same ParsedJson can be reused for other documents.
//
// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
@ -43,12 +44,12 @@ static inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool
// the input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING if reallocifneeded is false,
// all bytes at and after s.data()+s.size() are ignored (can be garbage).
WARN_UNUSED
static inline bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) {
inline bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) {
return json_parse(s.data(), s.size(), pj, reallocifneeded);
}
// Build a ParsedJson object. You can check validity
// Build a ParsedJson object. You can check validity
// by calling pj.isValid(). This does the memory allocation needed for ParsedJson.
// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
// (a copy of the input string is made).
@ -59,24 +60,26 @@ WARN_UNUSED
ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneeded = true);
WARN_UNUSED
// Build a ParsedJson object. You can check validity
// Build a ParsedJson object. You can check validity
// by calling pj.isValid(). This does the memory allocation needed for ParsedJson.
// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
// (a copy of the input string is made).
// The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
// all bytes at and after buf + len are ignored (can be garbage).
static inline ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded = true) {
inline ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded = true) {
return build_parsed_json((const uint8_t *) buf, len, reallocifneeded);
}
// convenience function
WARN_UNUSED
// Build a ParsedJson object. You can check validity
// Build a ParsedJson object. You can check validity
// by calling pj.isValid(). This does the memory allocation needed for ParsedJson.
// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
// (a copy of the input string is made).
// The input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING if reallocifneeded is false,
// all bytes at and after s.data()+s.size() are ignored (can be garbage).
static inline ParsedJson build_parsed_json(const std::string_view &s, bool reallocifneeded = true) {
inline ParsedJson build_parsed_json(const std::string_view &s, bool reallocifneeded = true) {
return build_parsed_json(s.data(), s.size(), reallocifneeded);
}
}
#endif

37
include/simdjson/numberparsing.h
View File

@ -1,5 +1,7 @@
#pragma once
#ifndef SIMDJSON_NUMBERPARSING_H
#define SIMDJSON_NUMBERPARSING_H
#include "simdjson/portability.h"
#include "simdjson/common_defs.h"
#include "simdjson/jsoncharutils.h"
#include "simdjson/parsedjson.h"
@ -105,10 +107,11 @@ is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) {
#ifdef SWAR_NUMBER_PARSING
#ifdef _MSC_VER
// check quickly whether the next 8 chars are made of digits
// at a glance, it looks better than Mula's
// http://0x80.pl/articles/swar-digits-validate.html
/*static inline bool is_made_of_eight_digits_fast(const char *chars) {
static inline bool is_made_of_eight_digits_fast(const char *chars) {
uint64_t val;
memcpy(&val, chars, 8);
// a branchy method might be faster:
@ -118,8 +121,8 @@ is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) {
return (((val & 0xF0F0F0F0F0F0F0F0) |
(((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) ==
0x3333333333333333);
}*/
}
#else
// this is more efficient apparently than the scalar code above (fewer instructions)
static inline bool is_made_of_eight_digits_fast(const char *chars) {
__m64 val;
@ -128,6 +131,7 @@ static inline bool is_made_of_eight_digits_fast(const char *chars) {
__m64 basecmp = _mm_subs_pu8(base,_mm_set1_pi8(9));
return _mm_cvtm64_si64(basecmp) == 0;
}
#endif
static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
// this actually computes *16* values so we are being wasteful.
@ -159,7 +163,7 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
// Note: a redesign could avoid this function entirely.
//
static never_inline bool
parse_float(const uint8_t *const buf,
parse_float(const uint8_t *const buf,
ParsedJson &pj, const uint32_t offset,
bool found_minus) {
const char *p = (const char *)(buf + offset);
@ -284,13 +288,13 @@ static never_inline bool parse_large_integer(const uint8_t *const buf,
// we rarely see large integer parts like 123456789
while (is_integer(*p)) {
digit = *p - '0';
if (__builtin_umulll_overflow(i, 10, (unsigned long long *)&i)) {
if (mul_overflow(i, 10, &i)) {
#ifdef JSON_TEST_NUMBERS // for unit testing
foundInvalidNumber(buf + offset);
#endif
return false; // overflow
}
if (__builtin_uaddll_overflow(i, digit, (unsigned long long *)&i)) {
if (add_overflow(i, digit, &i)) {
#ifdef JSON_TEST_NUMBERS // for unit testing
foundInvalidNumber(buf + offset);
#endif
@ -324,21 +328,14 @@ static never_inline bool parse_large_integer(const uint8_t *const buf,
return is_structural_or_whitespace(*p);
}
#ifndef likely
#define likely(x) __builtin_expect(!!(x), 1)
#endif
#ifndef unlikely
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
// parse the number at buf + offset
// define JSON_TEST_NUMBERS for unit testing
static really_inline bool parse_number(const uint8_t *const buf,
ParsedJson &pj,
const uint32_t offset,
static really_inline bool parse_number(const uint8_t *const buf,
ParsedJson &pj,
const uint32_t offset,
bool found_minus) {
#ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes useful to skip parsing
pj.write_tape_s64(0); // always write zero
@ -459,7 +456,7 @@ static really_inline bool parse_number(const uint8_t *const buf,
if (unlikely(digitcount >= 19)) { // this is uncommon!!!
// this is almost never going to get called!!!
// we start anew, going slowly!!!
return parse_float(buf, pj, offset,
return parse_float(buf, pj, offset,
found_minus);
}
///////////
@ -488,7 +485,7 @@ static really_inline bool parse_number(const uint8_t *const buf,
}
} else {
if (unlikely(digitcount >= 18)) { // this is uncommon!!!
return parse_large_integer(buf, pj, offset,
return parse_large_integer(buf, pj, offset,
found_minus);
}
pj.write_tape_s64(i);
@ -499,3 +496,5 @@ static really_inline bool parse_number(const uint8_t *const buf,
return is_structural_or_whitespace(*p);
#endif // SIMDJSON_SKIPNUMBERPARSING
}
#endif

160
include/simdjson/parsedjson.h
View File

@ -1,17 +1,13 @@
#pragma once
#include <math.h>
#ifndef SIMDJSON_PARSEDJSON_H
#define SIMDJSON_PARSEDJSON_H
#include <math.h>
#include <inttypes.h>
#include <string.h>
#ifdef _MSC_VER
/* Microsoft C/C++-compatible compiler */
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
#include <iomanip>
#include <iostream>
#include "simdjson/portability.h"
#include "simdjson/jsonformatutils.h"
#define JSONVALUEMASK 0xFFFFFFFFFFFFFF
@ -49,7 +45,8 @@ public:
}
isvalid = false;
bytecapacity = 0; // will only set it to len after allocations are a success
if (posix_memalign((void **)&structurals, 8, ROUNDUP_N(len, 64) / 8)) {
structurals = (uint8_t *)aligned_malloc(8, ROUNDUP_N(len, 64) / 8);
if (structurals == NULL) {
std::cerr << "Could not allocate memory for structurals" << std::endl;
return false;
};
@ -61,17 +58,20 @@ public:
string_buf = new uint8_t[localstringcapacity];
tape = new uint64_t[localtapecapacity];
containing_scope_offset = new uint32_t[maxdepth];
#ifdef SIMDJSON_USE_COMPUTED_GOTO
ret_address = new void *[maxdepth];
#else
ret_address = new char[maxdepth];
#endif
if ((string_buf == NULL) || (tape == NULL) ||
(containing_scope_offset == NULL) || (ret_address == NULL) || (structural_indexes == NULL)) {
std::cerr << "Could not allocate memory" << std::endl;
delete[] ret_address;
delete[] containing_scope_offset;
delete[] tape;
delete[] string_buf;
delete[] structural_indexes;
free(structurals);
if(ret_address != NULL) delete[] ret_address;
if(containing_scope_offset != NULL) delete[] containing_scope_offset;
if(tape != NULL) delete[] tape;
if(string_buf != NULL) delete[] string_buf;
if(structural_indexes != NULL) delete[] structural_indexes;
aligned_free(structurals);
return false;
}
@ -93,12 +93,12 @@ public:
depthcapacity = 0;
tapecapacity = 0;
stringcapacity = 0;
delete[] ret_address;
delete[] containing_scope_offset;
delete[] tape;
delete[] string_buf;
delete[] structural_indexes;
free(structurals);
if(ret_address != NULL) delete[] ret_address;
if(containing_scope_offset != NULL) delete[] containing_scope_offset;
if(tape != NULL) delete[] tape;
if(string_buf != NULL) delete[] string_buf;
if(structural_indexes != NULL) delete[] structural_indexes;
aligned_free(structurals);
isvalid = false;
}
@ -124,11 +124,11 @@ public:
if (type == 'r') {
howmany = tape_val & JSONVALUEMASK;
} else {
printf("Error: no starting root node?");
fprintf(stderr, "Error: no starting root node?");
return false;
}
if (howmany > tapecapacity) {
printf(
fprintf(stderr,
"We may be exceeding the tape capacity. Is this a valid document?\n");
return false;
}
@ -192,22 +192,22 @@ public:
os << '}';
break;
case '[': // we start an array
os << '[';
os << '[';
depth++;
inobject[depth] = false;
inobjectidx[depth] = 0;
break;
case ']': // we end an array
depth--;
os << ']';
os << ']';
break;
case 'r': // we start and end with the root node
printf("should we be hitting the root node?\n");
fprintf(stderr, "should we be hitting the root node?\n");
delete[] inobject;
delete[] inobjectidx;
return false;
default:
printf("bug %c\n", type);
fprintf(stderr, "bug %c\n", type);
delete[] inobject;
delete[] inobjectidx;
return false;
@ -224,25 +224,25 @@ public:
size_t tapeidx = 0;
uint64_t tape_val = tape[tapeidx];
uint8_t type = (tape_val >> 56);
os << tapeidx << " : " << type;
os << tapeidx << " : " << type;
tapeidx++;
size_t howmany = 0;
if (type == 'r') {
howmany = tape_val & JSONVALUEMASK;
} else {
printf("Error: no starting root node?");
fprintf(stderr, "Error: no starting root node?");
return false;
}
os << "\t// pointing to " << howmany <<" (right after last node)\n";
uint64_t payload;
for (; tapeidx < howmany; tapeidx++) {
os << tapeidx << " : ";
os << tapeidx << " : ";
tape_val = tape[tapeidx];
payload = tape_val & JSONVALUEMASK;
type = (tape_val >> 56);
switch (type) {
case '"': // we have a string
os << "string \"";
os << "string \"";
print_with_escapes((const unsigned char *)(string_buf + payload));
os << '"';
os << '\n';
@ -362,9 +362,9 @@ public:
delete[] depthindex;
}
iterator(const iterator &o):
pj(o.pj), depth(o.depth), location(o.location),
tape_length(o.tape_length), current_type(o.current_type),
iterator(const iterator &o):
pj(o.pj), depth(o.depth), location(o.location),
tape_length(o.tape_length), current_type(o.current_type),
current_val(o.current_val), depthindex(NULL) {
depthindex = new scopeindex_t[pj.depthcapacity];
if(depthindex != NULL) {
@ -374,18 +374,18 @@ public:
}
}
iterator(iterator &&o):
pj(o.pj), depth(o.depth), location(o.location),
tape_length(o.tape_length), current_type(o.current_type),
current_val(o.current_val), depthindex(o.depthindex) {
o.depthindex = NULL;// we take ownship
iterator(iterator &&o):
pj(o.pj), depth(std::move(o.depth)), location(std::move(o.location)),
tape_length(std::move(o.tape_length)), current_type(std::move(o.current_type)),
current_val(std::move(o.current_val)), depthindex(std::move(o.depthindex)) {
o.depthindex = NULL;// we take ownership
}
WARN_UNUSED
bool isOk() const {
return location < tape_length;
}
// useful for debuging purposes
size_t get_tape_location() const {
return location;
@ -417,7 +417,7 @@ public:
depth++;
depthindex[depth].start_of_scope = location;
depthindex[depth].scope_type = current_type;
}
}
location = location + 1;
current_val = pj.tape[location];
current_type = (current_val >> 56);
@ -456,7 +456,7 @@ public:
double answer;
memcpy(&answer, & pj.tape[location + 1], sizeof(answer));
return answer;
}
}
bool is_object_or_array() const {
return is_object_or_array(get_type());
@ -473,15 +473,15 @@ public:
bool is_string() const {
return get_type() == '"';
}
bool is_integer() const {
return get_type() == 'l';
}
bool is_double() const {
return get_type() == 'd';
}
static bool is_object_or_array(uint8_t type) {
return (type == '[' || (type == '{'));
}
@ -509,7 +509,7 @@ public:
really_inline const char * get_string() const {
return (const char *)(pj.string_buf + (current_val & JSONVALUEMASK)) ;
}
// throughout return true if we can do the navigation, false
// otherwise
@ -518,10 +518,10 @@ public:
// Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, { and [.
// At the object ({) or at the array ([), you can issue a "down" to visit their content.
// valid if we're not at the end of a scope (returns true).
really_inline bool next() {
really_inline bool next() {
if ((current_type == '[') || (current_type == '{')){
// we need to jump
size_t npos = ( current_val & JSONVALUEMASK);
size_t npos = ( current_val & JSONVALUEMASK);
if(npos >= tape_length) {
return false; // shoud never happen unless at the root
}
@ -548,14 +548,14 @@ public:
return true;
}
}
// Withing a given scope (series of nodes at the same depth within either an
// array or an object), we move backward.
// Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true when starting at the end
// of the scope.
// At the object ({) or at the array ([), you can issue a "down" to visit their content.
// At the object ({) or at the array ([), you can issue a "down" to visit their content.
really_inline bool prev() {
if(location - 1 < depthindex[depth].start_of_scope) return false;
location -= 1;
@ -563,9 +563,9 @@ public:
current_type = (current_val >> 56);
if ((current_type == ']') || (current_type == '}')){
// we need to jump
size_t new_location = ( current_val & JSONVALUEMASK);
size_t new_location = ( current_val & JSONVALUEMASK);
if(new_location < depthindex[depth].start_of_scope) {
return false; // shoud never happen
return false; // shoud never happen
}
location = new_location;
current_val = pj.tape[location];
@ -574,7 +574,7 @@ public:
return true;
}
// Moves back to either the containing array or object (type { or [) from
// Moves back to either the containing array or object (type { or [) from
// within a contained scope.
// Valid unless we are at the first level of the document
//
@ -590,8 +590,8 @@ public:
current_type = (current_val >> 56);
return true;
}
// Valid if we're at a [ or { and it starts a non-empty scope; moves us to start of
// that deeper scope if it not empty.
// Thus, given [true, null, {"a":1}, [1,2]], if we are at the { node, we would move to the
@ -599,7 +599,7 @@ public:
really_inline bool down() {
if(location + 1 >= tape_length) return false;
if ((current_type == '[') || (current_type == '{')) {
size_t npos = (current_val & JSONVALUEMASK);
size_t npos = (current_val & JSONVALUEMASK);
if(npos == location + 2) {
return false; // we have an empty scope
}
@ -610,13 +610,13 @@ public:
current_val = pj.tape[location];
current_type = (current_val >> 56);
return true;
}
}
return false;
}
// move us to the start of our current scope,
// a scope is a series of nodes at the same level
void to_start_scope() {
void to_start_scope() {
location = depthindex[depth].start_of_scope;
current_val = pj.tape[location];
current_type = (current_val >> 56);
@ -641,7 +641,7 @@ public:
case 'l': // we have a long int
os << get_integer();
break;
case 'd':
case 'd':
os << get_double();
break;
case 'n': // we have a null
@ -674,7 +674,7 @@ private:
ParsedJson &pj;
size_t depth;
size_t location; // our current location on a tape
size_t tape_length;
size_t tape_length;
uint8_t current_type;
uint64_t current_val;
scopeindex_t *depthindex;
@ -696,12 +696,39 @@ private:
uint64_t *tape;
uint32_t *containing_scope_offset;
#ifdef SIMDJSON_USE_COMPUTED_GOTO
void **ret_address;
#else
char *ret_address;
#endif
uint8_t *string_buf; // should be at least bytecapacity
uint8_t *current_string_buf_loc;
bool isvalid;
ParsedJson(const ParsedJson && p);
ParsedJson(ParsedJson && p)
: bytecapacity(std::move(p.bytecapacity)),
depthcapacity(std::move(p.depthcapacity)),
tapecapacity(std::move(p.tapecapacity)),
stringcapacity(std::move(p.stringcapacity)),
current_loc(std::move(p.current_loc)),
structurals(std::move(p.structurals)),
n_structural_indexes(std::move(p.n_structural_indexes)),
structural_indexes(std::move(p.structural_indexes)),
tape(std::move(p.tape)),
containing_scope_offset(std::move(p.containing_scope_offset)),
ret_address(std::move(p.ret_address)),
string_buf(std::move(p.string_buf)),
current_string_buf_loc(std::move(p.current_string_buf_loc)),
isvalid(std::move(p.isvalid)) {
p.structurals=NULL;
p.structural_indexes=NULL;
p.tape=NULL;
p.containing_scope_offset=NULL;
p.ret_address=NULL;
p.string_buf=NULL;
p.current_string_buf_loc=NULL;
}
private :
@ -719,12 +746,15 @@ inline void dumpbits_always(uint64_t v, const std::string &msg) {
for (uint32_t i = 0; i < 64; i++) {
std::cout << (((v >> (uint64_t)i) & 0x1ULL) ? "1" : "_");
}
std::cout << " " << msg << "\n";
std::cout << " " << msg.c_str() << "\n";
}
inline void dumpbits32_always(uint32_t v, const std::string &msg) {
for (uint32_t i = 0; i < 32; i++) {
std::cout << (((v >> (uint32_t)i) & 0x1ULL) ? "1" : "_");
}
std::cout << " " << msg << "\n";
std::cout << " " << msg.c_str() << "\n";
}
#endif

122
include/simdjson/portability.h Normal file
View File

@ -0,0 +1,122 @@
#ifndef SIMDJSON_PORTABILITY_H
#define SIMDJSON_PORTABILITY_H
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
#ifdef _MSC_VER
/* Microsoft C/C++-compatible compiler */
#include <intrin.h>
#include <iso646.h>
#include <cstdint>
static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
return _addcarry_u64(0, value1, value2, reinterpret_cast<unsigned __int64 *>(result));
}
# pragma intrinsic(_umul128)
static inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
uint64_t high;
*result = _umul128(value1, value2, &high);
return high;
}
static inline int trailingzeroes(uint64_t input_num) {
return _tzcnt_u64(input_num);
}
static inline int leadingzeroes(uint64_t input_num) {
return _lzcnt_u64(input_num);
}
static inline int hamming(uint64_t input_num) {
#ifdef _WIN64 // highly recommended!!!
return (int)__popcnt64(input_num);
#else // if we must support 32-bit Windows
return (int)(__popcnt((uint32_t)input_num) +
__popcnt((uint32_t)(input_num >> 32)));
#endif
}
#else
#include <x86intrin.h>
#include <cstdint>
static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
return __builtin_uaddll_overflow(value1, value2, (unsigned long long*)result);
}
static inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
return __builtin_umulll_overflow(value1, value2, (unsigned long long *)result);
}
/* result might be undefined when input_num is zero */
static inline int trailingzeroes(uint64_t input_num) {
return _tzcnt_u64(input_num);
}
/* result might be undefined when input_num is zero */
static inline int leadingzeroes(uint64_t input_num) {
return _lzcnt_u64(input_num);
}
/* result might be undefined when input_num is zero */
static inline int hamming(uint64_t input_num) {
return _popcnt64(input_num);
}
#endif // _MSC_VER
// portable version of posix_memalign
static inline void *aligned_malloc(size_t alignment, size_t size) {
void *p;
#ifdef _MSC_VER
p = _aligned_malloc(size, alignment);
#elif defined(__MINGW32__) || defined(__MINGW64__)
p = __mingw_aligned_malloc(size, alignment);
#else
// somehow, if this is used before including "x86intrin.h", it creates an
// implicit defined warning.
if (posix_memalign(&p, alignment, size) != 0) return NULL;
#endif
return p;
}
#ifndef __clang__
#ifndef _MSC_VER
static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi,
__m128i const *__addr_lo) {
__m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo));
return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1);
}
static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo,
__m256i __a) {
__m128i __v128;
__v128 = _mm256_castsi256_si128(__a);
_mm_storeu_si128(__addr_lo, __v128);
__v128 = _mm256_extractf128_si256(__a, 1);
_mm_storeu_si128(__addr_hi, __v128);
}
#endif
#endif
static inline void aligned_free(void *memblock) {
if(memblock == NULL) return;
#ifdef _MSC_VER
_aligned_free(memblock);
#elif defined(__MINGW32__) || defined(__MINGW64__)
__mingw_aligned_free(memblock);
#else
free(memblock);
#endif
}
#endif /* end of include PORTABILITY_H */

18
include/simdjson/simdprune_tables.h
View File

@ -1,19 +1,9 @@
/**
* (c) Daniel Lemire
* License: Apache License 2.0
*/
#ifndef SIMDJSON_SIMDPRUNE_TABLES_H
#define SIMDJSON_SIMDPRUNE_TABLES_H
#ifndef SIMDPRUNE_TABLES_H
#define SIMDPRUNE_TABLES_H
#include "simdjson/portability.h"
#if defined(_MSC_VER)
#include <intrin.h>
#include <iso646.h>
#else
#include <x86intrin.h>
#endif
#include <cstdint>
#ifdef __SSE3__
#ifdef __AVX__
static const unsigned char mask128_epi8[] = {
0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe,

12
include/simdjson/simdutf8check.h
View File

@ -1,16 +1,12 @@
#ifndef SIMDUTF8CHECK_H
#define SIMDUTF8CHECK_H
#ifndef SIMDJSON_SIMDUTF8CHECK_H
#define SIMDJSON_SIMDUTF8CHECK_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#ifdef _MSC_VER
/* Microsoft C/C++-compatible compiler */
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
#include <string.h>
#include "simdjson/portability.h"
/*
* legal utf-8 byte sequence
* http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94

9
include/simdjson/stage1_find_marks.h
View File

@ -1,7 +1,8 @@
#pragma once
#ifndef SIMDJSON_STAGE1_FIND_MARKS_H
#define SIMDJSON_STAGE1_FIND_MARKS_H
#include "common_defs.h"
#include "parsedjson.h"
#include "simdjson/common_defs.h"
#include "simdjson/parsedjson.h"
WARN_UNUSED
bool find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj);
@ -10,3 +11,5 @@ WARN_UNUSED
static inline bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj) {
return find_structural_bits((const uint8_t *)buf, len, pj);
}
#endif

5
include/simdjson/stage2_flatten.h
View File

@ -1,7 +1,10 @@
#pragma once
#ifndef SIMDJSON_STAGE2_FLATTEN_H
#define SIMDJSON_STAGE2_FLATTEN_H
#include "simdjson/common_defs.h"
#include "simdjson/parsedjson.h"
WARN_UNUSED
bool flatten_indexes(size_t len, ParsedJson &pj);
#endif

4
include/simdjson/stage34_unified.h
View File

@ -1,4 +1,5 @@
#pragma once
#ifndef SIMDJSON_STAGE34_UNIFIED_H
#define SIMDJSON_STAGE34_UNIFIED_H
#include "simdjson/common_defs.h"
#include "simdjson/parsedjson.h"
@ -13,3 +14,4 @@ static inline bool unified_machine(const char *buf, size_t len, ParsedJson &pj)
return unified_machine((const uint8_t *)buf,len,pj);
}
#endif

12
include/simdjson/stringparsing.h
View File

@ -1,4 +1,5 @@
#pragma once
#ifndef SIMDJSON_STRINGPARSING_H
#define SIMDJSON_STRINGPARSING_H
#include "simdjson/common_defs.h"
#include "simdjson/parsedjson.h"
@ -53,7 +54,7 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **d
(((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000;
*src_ptr += 6;
}
size_t offset = codepoint_to_utf8(code_point, *dst_ptr);
size_t offset = codepoint_to_utf8(code_point, *dst_ptr);
*dst_ptr += offset;
return offset > 0;
}
@ -87,8 +88,8 @@ really_inline bool parse_string(const uint8_t *buf, UNUSED size_t len,
__m256i unescaped_vec = _mm256_cmpeq_epi8(_mm256_max_epu8(unitsep,v),unitsep);// could do it with saturated subtraction
#endif // CHECKUNESCAPED
uint32_t quote_dist = __tzcnt_u64(quote_bits);
uint32_t bs_dist = __tzcnt_u64(bs_bits);
uint32_t quote_dist = trailingzeroes(quote_bits);
uint32_t bs_dist = trailingzeroes(bs_bits);
// store to dest unconditionally - we can overwrite the bits we don't like
// later
_mm256_storeu_si256((__m256i *)(dst), v);
@ -104,7 +105,7 @@ really_inline bool parse_string(const uint8_t *buf, UNUSED size_t len,
uint32_t unescaped_bits = (uint32_t)_mm256_movemask_epi8(unescaped_vec);
bool is_ok = ((quote_bits - 1) & (~ quote_bits) & unescaped_bits) == 0;
#ifdef JSON_TEST_STRINGS // for unit testing
if(is_ok) foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1);
if(is_ok) foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1);
else foundBadString(buf + offset);
#endif // JSON_TEST_STRINGS
return is_ok;
@ -176,3 +177,4 @@ really_inline bool parse_string(const uint8_t *buf, UNUSED size_t len,
}
#endif

16
src/jsonioutil.cpp
View File

@ -1,6 +1,6 @@
#include "simdjson/jsonioutil.h"
#include <cstring>
#include <stdlib.h>
char * allocate_padded_buffer(size_t length) {
// we could do a simple malloc
@ -8,10 +8,14 @@ char * allocate_padded_buffer(size_t length) {
// However, we might as well align to cache lines...
char *padded_buffer;
size_t totalpaddedlength = length + SIMDJSON_PADDING;
if (posix_memalign((void **)&padded_buffer, 64, totalpaddedlength)) {
return NULL;
};
return padded_buffer;
#ifdef _MSC_VER
padded_buffer = (char*) _aligned_malloc(totalpaddedlength, 64);
#elif defined(__MINGW32__) || defined(__MINGW64__)
padded_buffer = __mingw_aligned_malloc(totalpaddedlength, 64);
#else
if (posix_memalign((void **)&padded_buffer, 64, totalpaddedlength) != 0) return NULL;
#endif
return padded_buffer;
}
std::string_view get_corpus(std::string filename) {
@ -28,7 +32,7 @@ std::string_view get_corpus(std::string filename) {
size_t readb = std::fread(buf, 1, len, fp);
std::fclose(fp);
if(readb != len) {
free(buf);
aligned_free(buf);
throw std::runtime_error("could not read the data");
}
return std::string_view(buf,len);

49
src/jsonminifier.cpp
View File

@ -1,7 +1,7 @@
#include <cstdint>
#include "simdjson/portability.h"
#ifndef __AVX2__
#include <cstdint>
static uint8_t jump_table[256 * 3] = {
0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0,
@ -59,33 +59,8 @@ size_t jsonminify(const unsigned char *bytes, size_t howmany,
#else
#ifdef _MSC_VER
/* Microsoft C/C++-compatible compiler */
#include <intrin.h>
#else
#include <immintrin.h>
#include <x86intrin.h>
#endif // _MSC_VER
#include "simdjson/simdprune_tables.h"
#include <cstring>
#ifndef __clang__
static inline __m256i _mm256_loadu2_m128i(__m128i const *__addr_hi,
__m128i const *__addr_lo) {
__m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo));
return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1);
}
static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo,
__m256i __a) {
__m128i __v128;
__v128 = _mm256_castsi256_si128(__a);
_mm_storeu_si128(__addr_lo, __v128);
__v128 = _mm256_extractf128_si256(__a, 1);
_mm_storeu_si128(__addr_hi, __v128);
}
#endif
// a straightforward comparison of a mask against input.
static uint64_t cmp_mask_against_input_mini(__m256i input_lo, __m256i input_hi,
@ -122,7 +97,7 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) {
uint64_t odd_starts = start_edges & ~even_start_mask;
uint64_t even_carries = bs_bits + even_starts;
uint64_t odd_carries;
bool iter_ends_odd_backslash = __builtin_uaddll_overflow(
bool iter_ends_odd_backslash = add_overflow(
bs_bits, odd_starts, (unsigned long long *)&odd_carries);
odd_carries |= prev_iter_ends_odd_backslash;
prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL;
@ -171,10 +146,10 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) {
int mask2 = (whitespace >> 16) & 0xFFFF;
int mask3 = (whitespace >> 32) & 0xFFFF;
int mask4 = (whitespace >> 48) & 0xFFFF;
int pop1 = _popcnt64((~whitespace) & 0xFFFF);
int pop2 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFF));
int pop3 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFFFFFF));
int pop4 = _popcnt64((~whitespace));
int pop1 = hamming((~whitespace) & 0xFFFF);
int pop2 = hamming((~whitespace) & UINT64_C(0xFFFFFFFF));
int pop3 = hamming((~whitespace) & UINT64_C(0xFFFFFFFFFFFF));
int pop4 = hamming((~whitespace));
__m256i vmask1 =
_mm256_loadu2_m128i((const __m128i *)mask128_epi8 + (mask2 & 0x7FFF),
(const __m128i *)mask128_epi8 + (mask1 & 0x7FFF));
@ -206,7 +181,7 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) {
uint64_t even_carries = bs_bits + even_starts;
uint64_t odd_carries;
//bool iter_ends_odd_backslash =
__builtin_uaddll_overflow( bs_bits, odd_starts, (unsigned long long *)&odd_carries);
add_overflow( bs_bits, odd_starts, &odd_carries);
odd_carries |= prev_iter_ends_odd_backslash;
//prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL; // we never use it
uint64_t even_carry_ends = even_carries & ~bs_bits;
@ -250,10 +225,10 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) {
int mask2 = (whitespace >> 16) & 0xFFFF;
int mask3 = (whitespace >> 32) & 0xFFFF;
int mask4 = (whitespace >> 48) & 0xFFFF;
int pop1 = _popcnt64((~whitespace) & 0xFFFF);
int pop2 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFF));
int pop3 = _popcnt64((~whitespace) & UINT64_C(0xFFFFFFFFFFFF));
int pop4 = _popcnt64((~whitespace));
int pop1 = hamming((~whitespace) & 0xFFFF);
int pop2 = hamming((~whitespace) & UINT64_C(0xFFFFFFFF));
int pop3 = hamming((~whitespace) & UINT64_C(0xFFFFFFFFFFFF));
int pop4 = hamming((~whitespace));
__m256i vmask1 =
_mm256_loadu2_m128i((const __m128i *)mask128_epi8 + (mask2 & 0x7FFF),
(const __m128i *)mask128_epi8 + (mask1 & 0x7FFF));

22
src/jsonparser.cpp
View File

@ -1,5 +1,17 @@
#include "simdjson/jsonparser.h"
#ifdef _MSC_VER
#include <windows.h>
#include <sysinfoapi.h>
#else
#include <unistd.h>
#endif
extern bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded);
extern bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded);
extern ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded);
extern ParsedJson build_parsed_json(const std::string_view &s, bool reallocifneeded);
// parse a document found in buf, need to preallocate ParsedJson.
WARN_UNUSED
@ -12,8 +24,14 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne
bool reallocated = false;
if(reallocifneeded) {
// realloc is needed if the end of the memory crosses a page
long pagesize = sysconf (_SC_PAGESIZE); // on windows this should be SYSTEM_INFO sysInfo; GetSystemInfo(&sysInfo); sysInfo.dwPageSize
if ( (reinterpret_cast<uintptr_t>(buf + len - 1) % pagesize ) < SIMDJSON_PADDING ) {
#ifdef _MSC_VER
SYSTEM_INFO sysInfo;
GetSystemInfo(&sysInfo);
long pagesize = sysInfo.dwPageSize;
#else
long pagesize = sysconf (_SC_PAGESIZE);
#endif
if ( (reinterpret_cast<uintptr_t>(buf + len - 1) % pagesize ) < SIMDJSON_PADDING ) {
const uint8_t *tmpbuf = buf;
buf = (uint8_t *) allocate_padded_buffer(len);
if(buf == NULL) return false;

35
src/stage1_find_marks.cpp
View File

@ -1,12 +1,5 @@
#ifdef _MSC_VER
/* Microsoft C/C++-compatible compiler */
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
#include "simdjson/portability.h"
#include <cassert>
#include "simdjson/common_defs.h"
#include "simdjson/parsedjson.h"
@ -16,7 +9,7 @@
// It seems that many parsers do UTF-8 validation.
// RapidJSON does not do it by default, but a flag
// allows it.
// allows it.
#ifdef SIMDJSON_UTF8VALIDATE
#include "simdjson/simdutf8check.h"
#endif
@ -42,10 +35,10 @@ WARN_UNUSED
}
#ifdef SIMDJSON_UTF8VALIDATE
__m256i has_error = _mm256_setzero_si256();
struct avx_processed_utf_bytes previous = {
.rawbytes = _mm256_setzero_si256(),
.high_nibbles = _mm256_setzero_si256(),
.carried_continuations = _mm256_setzero_si256()};
struct avx_processed_utf_bytes previous;
previous.rawbytes = _mm256_setzero_si256();
previous.high_nibbles = _mm256_setzero_si256();
previous.carried_continuations = _mm256_setzero_si256();
#endif
// Useful constant masks
@ -63,11 +56,13 @@ WARN_UNUSED
// effectively the very first char is considered to follow "whitespace" for the
// purposes of psuedo-structural character detection
uint64_t prev_iter_ends_pseudo_pred = 1ULL;
size_t lenminus64 = len < 64 ? 0 : len - 64;
size_t lenminus64 = len < 64 ? 0 : len - 64;
size_t idx = 0;
for (; idx < lenminus64; idx += 64) {
#ifndef _MSC_VER
__builtin_prefetch(buf + idx + 128);
__m256i input_lo = _mm256_loadu_si256((const __m256i *)(buf + idx + 0));
#endif
__m256i input_lo = _mm256_loadu_si256((const __m256i *)(buf + idx + 0));
__m256i input_hi = _mm256_loadu_si256((const __m256i *)(buf + idx + 32));
#ifdef SIMDJSON_UTF8VALIDATE
__m256i highbit = _mm256_set1_epi8(0x80);
@ -78,7 +73,7 @@ WARN_UNUSED
_mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 1)),has_error);
} else {
// it is not ascii so we have to do heavy work
previous = avxcheckUTF8Bytes(input_lo, &previous, &has_error);
@ -104,7 +99,7 @@ WARN_UNUSED
// indicates whether the sense of any edge going to the next iteration
// should be flipped
bool iter_ends_odd_backslash =
__builtin_uaddll_overflow(bs_bits, odd_starts, (unsigned long long *) &odd_carries);
add_overflow(bs_bits, odd_starts, &odd_carries);
odd_carries |=
prev_iter_ends_odd_backslash; // push in bit zero as a potential end
@ -214,7 +209,7 @@ WARN_UNUSED
/// but otherwise the string needs to be properly padded or else we
/// risk invalidating the UTF-8 checks.
////////////
if (idx < len) {
if (idx < len) {
uint8_t tmpbuf[64];
memset(tmpbuf,0x20,64);
memcpy(tmpbuf,buf+idx,len - idx);
@ -229,7 +224,7 @@ WARN_UNUSED
_mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9, 9, 1)),has_error);
} else {
// it is not ascii so we have to do heavy work
previous = avxcheckUTF8Bytes(input_lo, &previous, &has_error);
@ -255,7 +250,7 @@ WARN_UNUSED
// indicates whether the sense of any edge going to the next iteration
// should be flipped
//bool iter_ends_odd_backslash =
__builtin_uaddll_overflow(bs_bits, odd_starts, (unsigned long long *) &odd_carries);
add_overflow(bs_bits, odd_starts, &odd_carries);
odd_carries |=
prev_iter_ends_odd_backslash; // push in bit zero as a potential end

32
src/stage2_flatten.cpp
View File

@ -1,12 +1,6 @@
#ifdef _MSC_VER
/* Microsoft C/C++-compatible compiler */
#include <intrin.h>
#else
#include <x86intrin.h>
#endif
#include <cassert>
#include "simdjson/portability.h"
#include "simdjson/common_defs.h"
#include "simdjson/parsedjson.h"
@ -20,7 +14,7 @@
#endif
#define SET_BIT(i) \
base_ptr[base + i] = (uint32_t)idx + __tzcnt_u64(s); \
base_ptr[base + i] = (uint32_t)idx + trailingzeroes(s); \
s = s & (s - 1);
#define SET_BIT1 SET_BIT(0)
@ -59,7 +53,7 @@ bool flatten_indexes(size_t len, ParsedJson &pj) {
counters[k] = 0;
for (size_t idx = 0; idx < len; idx += 64) {
uint64_t s = *(uint64_t *)(pj.structurals + idx / 8);
uint32_t cnt = __builtin_popcountll(s);
uint32_t cnt = hamming(s);
total++;
counters[cnt]++;
}
@ -74,23 +68,23 @@ bool flatten_indexes(size_t len, ParsedJson &pj) {
uint64_t s = *(uint64_t *)(pj.structurals + idx / 8);
#ifdef SUPPRESS_CHEESY_FLATTEN
while (s) {
base_ptr[base++] = (uint32_t)idx + __builtin_ctzll(s);
base_ptr[base++] = (uint32_t)idx + trailingzeroes(s);
s &= s - 1ULL;
}
#elif defined(NO_PDEP_PLEASE)
uint32_t cnt = __builtin_popcountll(s);
uint32_t cnt = hamming(s);
uint32_t next_base = base + cnt;
while (s) {
CALL(SET_BITLOOPN, NO_PDEP_WIDTH)
/*for(size_t i = 0; i < NO_PDEP_WIDTH; i++) {
base_ptr[base+i] = (uint32_t)idx + __builtin_ctzll(s);
base_ptr[base+i] = (uint32_t)idx + trailingzeroes(s);
s = s & (s - 1);
}*/
base += NO_PDEP_WIDTH;
}
base = next_base;
#else
uint32_t cnt = __builtin_popcountll(s);
uint32_t cnt = hamming(s);
uint32_t next_base = base + cnt;
while (s) {
// spoil the suspense by reducing dependency chains; actually a win even
@ -98,18 +92,18 @@ bool flatten_indexes(size_t len, ParsedJson &pj) {
uint64_t s3 = _pdep_u64(~0x7ULL, s); // s3 will have bottom 3 1-bits unset
uint64_t s5 = _pdep_u64(~0x1fULL, s); // s5 will have bottom 5 1-bits unset
base_ptr[base + 0] = (uint32_t)idx + __builtin_ctzll(s);
base_ptr[base + 0] = (uint32_t)idx + trailingzeroes(s);
uint64_t s1 = s & (s - 1ULL);
base_ptr[base + 1] = (uint32_t)idx + __builtin_ctzll(s1);
base_ptr[base + 1] = (uint32_t)idx + trailingzeroes(s1);
uint64_t s2 = s1 & (s1 - 1ULL);
base_ptr[base + 2] =
(uint32_t)idx + __builtin_ctzll(s2); // uint64_t s3 = s2 & (s2 - 1ULL);
base_ptr[base + 3] = (uint32_t)idx + __builtin_ctzll(s3);
(uint32_t)idx + trailingzeroes(s2); // uint64_t s3 = s2 & (s2 - 1ULL);
base_ptr[base + 3] = (uint32_t)idx + trailingzeroes(s3);
uint64_t s4 = s3 & (s3 - 1ULL);
base_ptr[base + 4] =
(uint32_t)idx + __builtin_ctzll(s4); // uint64_t s5 = s4 & (s4 - 1ULL);
base_ptr[base + 5] = (uint32_t)idx + __builtin_ctzll(s5);
(uint32_t)idx + trailingzeroes(s4); // uint64_t s5 = s4 & (s4 - 1ULL);
base_ptr[base + 5] = (uint32_t)idx + trailingzeroes(s5);
uint64_t s6 = s5 & (s5 - 1ULL);
s = s6;
base += 6;

45
src/stage34_unified.cpp
View File

@ -61,9 +61,6 @@ really_inline bool is_valid_null_atom(const uint8_t *loc) {
* The JSON is parsed to a tape, see the accompanying tape.md file
* for documentation.
***********/
// Implemented using Labels as Values which works in GCC and CLANG (and maybe
// also in Intel's compiler), but won't work in MSVC. This would need to be
// reimplemented differently if one wants to be standard compliant.
WARN_UNUSED
bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
uint32_t i = 0; // index of the structural character (0,1,2,3...)
@ -73,7 +70,7 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
uint32_t depth = 0; // could have an arbitrary starting depth
pj.init();
if(pj.bytecapacity < len) {
printf("insufficient capacity\n");
fprintf(stderr, "insufficient capacity\n");
return false;
}
// this macro reads the next structural character, updating idx, i and c.
@ -85,7 +82,11 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
////////////////////////////// START STATE /////////////////////////////
#ifdef SIMDJSON_USE_COMPUTED_GOTO
pj.ret_address[depth] = &&start_continue;
#else
pj.ret_address[depth] = 's';
#endif
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, 'r'); // r for root, 0 is going to get overwritten
// the root is used, if nothing else, to capture the size of the tape
@ -98,7 +99,11 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
switch (c) {
case '{':
pj.containing_scope_offset[depth] = pj.get_current_loc();
#ifdef SIMDJSON_USE_COMPUTED_GOTO
pj.ret_address[depth] = &&start_continue;
#else
pj.ret_address[depth] = 's';
#endif
depth++;
if (depth > pj.depthcapacity) {
goto fail;
@ -107,7 +112,11 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
goto object_begin;
case '[':
pj.containing_scope_offset[depth] = pj.get_current_loc();
#ifdef SIMDJSON_USE_COMPUTED_GOTO
pj.ret_address[depth] = &&start_continue;
#else
pj.ret_address[depth] = 's';
#endif
depth++;
if (depth > pj.depthcapacity) {
goto fail;
@ -299,7 +308,11 @@ object_key_state:
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized
// we have not yet encountered } so we need to come back for it
#ifdef SIMDJSON_USE_COMPUTED_GOTO
pj.ret_address[depth] = &&object_continue;
#else
pj.ret_address[depth] = 'o';
#endif
// we found an object inside an object, so we need to increment the depth
depth++;
if (depth > pj.depthcapacity) {
@ -312,7 +325,11 @@ object_key_state:
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized
// we have not yet encountered } so we need to come back for it
#ifdef SIMDJSON_USE_COMPUTED_GOTO
pj.ret_address[depth] = &&object_continue;
#else
pj.ret_address[depth] = 'o';
#endif
// we found an array inside an object, so we need to increment the depth
depth++;
if (depth > pj.depthcapacity) {
@ -352,7 +369,15 @@ scope_end:
pj.annotate_previousloc(pj.containing_scope_offset[depth],
pj.get_current_loc());
// goto saved_state
#ifdef SIMDJSON_USE_COMPUTED_GOTO
goto *pj.ret_address[depth];
#else
if(pj.ret_address[depth] == 'a') {
goto array_continue;
} else if (pj.ret_address[depth] == 'o') {
goto object_continue;
} else goto start_continue;
#endif
////////////////////////////// ARRAY STATES /////////////////////////////
array_begin:
@ -415,7 +440,11 @@ main_array_switch:
// we have not yet encountered ] so we need to come back for it
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized
#ifdef SIMDJSON_USE_COMPUTED_GOTO
pj.ret_address[depth] = &&array_continue;
#else
pj.ret_address[depth] = 'a';
#endif
// we found an object inside an array, so we need to increment the depth
depth++;
if (depth > pj.depthcapacity) {
@ -428,7 +457,11 @@ main_array_switch:
// we have not yet encountered ] so we need to come back for it
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized
#ifdef SIMDJSON_USE_COMPUTED_GOTO
pj.ret_address[depth] = &&array_continue;
#else
pj.ret_address[depth] = 'a';
#endif
// we found an array inside an array, so we need to increment the depth
depth++;
if (depth > pj.depthcapacity) {
@ -457,11 +490,11 @@ array_continue:
succeed:
depth --;
if(depth != 0) {
printf("internal bug\n");
fprintf(stderr, "internal bug\n");
abort();
}
if(pj.containing_scope_offset[depth] != 0) {
printf("internal bug\n");
fprintf(stderr, "internal bug\n");
abort();
}
pj.annotate_previousloc(pj.containing_scope_offset[depth],

5
tests/CMakeLists.txt
View File

@ -1,3 +1,8 @@
if(MSVC)
target_include_directories(${SIMDJSON_LIB_NAME}
PUBLIC ${PROJECT_SOURCE_DIR}/windows
)
endif()
add_cpp_test(jsoncheck)
add_test(jsoncheck jsoncheck)

2
tests/allparserscheckfile.cpp
View File

@ -108,7 +108,7 @@ int main(int argc, char *argv[]) {
printf("fastjson : %s \n", fastjson_correct ? "correct":"invalid");
printf("gason : %s \n", gason_correct ? "correct":"invalid");
printf("ultrajson : %s \n", ultrajson_correct ? "correct":"invalid");
free((void*)p.data());
aligned_free((void*)p.data());
free(buffer);
return EXIT_SUCCESS;
}

24
tests/jsoncheck.cpp
View File

@ -1,11 +1,16 @@
#include <assert.h>
#include <cstring>
#ifndef _MSC_VER
#include <dirent.h>
#include <unistd.h>
#else
// Microsoft can't be bothered to provide standard utils.
#include <dirent_portable.h>
#endif
#include <inttypes.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "simdjson/jsonparser.h"
@ -34,7 +39,7 @@ bool validate(const char *dirname) {
struct dirent **entry_list;
int c = scandir(dirname, &entry_list, 0, alphasort);
if (c < 0) {
printf("error accessing %s \n", dirname);
fprintf(stderr, "error accessing %s \n", dirname);
return false;
}
if (c == 0) {
@ -62,19 +67,19 @@ bool validate(const char *dirname) {
std::string_view p;
try {
p = get_corpus(fullpath);
} catch (const std::exception& e) {
std::cout << "Could not load the file " << fullpath << std::endl;
} catch (const std::exception& e) {
std::cerr << "Could not load the file " << fullpath << std::endl;
return EXIT_FAILURE;
}
ParsedJson pj;
bool allocok = pj.allocateCapacity(p.size(), 1024);
if(!allocok) {
std::cerr<< "can't allocate memory"<<std::endl;
std::cerr << "can't allocate memory"<<std::endl;
return false;
}
++howmany;
bool isok = json_parse(p, pj);
free((void*)p.data());
aligned_free((void*)p.data());
printf("%s\n", isok ? "ok" : "invalid");
if(contains("EXCLUDE",name)) {
// skipping
@ -91,7 +96,7 @@ bool validate(const char *dirname) {
printf("warning: file %s should fail but it passes.\n", name);
everythingfine = false;
}
}
}
free(fullpath);
}
}
@ -99,16 +104,15 @@ bool validate(const char *dirname) {
if(everythingfine) {
printf("All ok!\n");
} else {
printf("There were problems! Consider reviewing the following files:\n");
fprintf(stderr, "There were problems! Consider reviewing the following files:\n");
for(int i = 0; i < c; i++) {
if(!isfileasexpected[i]) printf("%s \n", entry_list[i]->d_name);
if(!isfileasexpected[i]) fprintf(stderr, "%s \n", entry_list[i]->d_name);
}
}
for (int i = 0; i < c; ++i)
free(entry_list[i]);
free(entry_list);
delete[] isfileasexpected;
return everythingfine;
}

22
tests/numberparsingcheck.cpp
View File

@ -57,8 +57,8 @@ inline void foundInteger(int64_t result, const uint8_t *buf) {
char *endptr;
long long expected = strtoll((const char *)buf, &endptr, 10);
if ((endptr == (const char *)buf) || (expected != result)) {
printf("Error: parsed %" PRId64 " out of %.32s, ", result, buf);
printf(" while parsing %s \n", fullpath);
fprintf(stderr, "Error: parsed %" PRId64 " out of %.32s, ", result, buf);
fprintf(stderr, " while parsing %s \n", fullpath);
parse_error |= PARSE_ERROR;
}
}
@ -68,23 +68,23 @@ inline void foundFloat(double result, const uint8_t *buf) {
float_count++;
double expected = strtod((const char *)buf, &endptr);
if (endptr == (const char *)buf) {
printf("parsed %f from %.32s whereas strtod refuses to parse a float, ",
fprintf(stderr, "parsed %f from %.32s whereas strtod refuses to parse a float, ",
result, buf);
printf(" while parsing %s \n", fullpath);
fprintf(stderr, " while parsing %s \n", fullpath);
parse_error |= PARSE_ERROR;
}
if( fpclassify(expected) != fpclassify(result) ) {
printf("floats not in the same category expected: %f observed: %f \n", expected, result);
printf("%.128s\n", buf);
fprintf(stderr, "floats not in the same category expected: %f observed: %f \n", expected, result);
fprintf(stderr, "%.128s\n", buf);
parse_error |= PARSE_ERROR;
}
// we want to get some reasonable relative accuracy
else if (fabs(expected - result) / fmin(fabs(expected), fabs(result)) >
1e-14) {
printf("parsed %.128e from \n", result);
printf(" %.200s whereas strtod gives\n", buf);
printf(" %.128e,", expected);
printf(" while parsing %s \n", fullpath);
fprintf(stderr, "parsed %.128e from \n", result);
fprintf(stderr, " %.200s whereas strtod gives\n", buf);
fprintf(stderr, " %.128e,", expected);
fprintf(stderr, " while parsing %s \n", fullpath);
parse_error |= PARSE_ERROR;
}
}
@ -154,7 +154,7 @@ bool validate(const char *dirname) {
float_count, invalid_count,
int_count + float_count + invalid_count);
}
free((void*)p.data());
aligned_free((void*)p.data());
free(fullpath);
}
}

6
tests/stringparsingcheck.cpp
View File

@ -341,7 +341,7 @@ bool validate(const char *dirname) {
bigbuffer = (char *) malloc(p.size());
if(bigbuffer == NULL) {
std::cerr << "can't allocate memory" << std::endl;
free((void*)p.data());
aligned_free((void*)p.data());
return false;
}
bad_string = 0;
@ -350,7 +350,7 @@ bool validate(const char *dirname) {
empty_string = 0;
bool isok = json_parse(p, pj);
free(bigbuffer);
free((void*)p.data());
aligned_free((void*)p.data());
if (good_string > 0) {
printf("File %40s %s --- bad strings: %10zu \tgood strings: %10zu\t "
"empty strings: %10zu "
@ -368,7 +368,7 @@ bool validate(const char *dirname) {
}
printf("%zu strings checked.\n", total_strings);
if (probable_bug) {
printf("STRING PARSING FAILS?\n");
fprintf(stderr, "STRING PARSING FAILS?\n");
} else {
printf("All ok.\n");
}

5
tools/cmake/FindOptions.cmake
View File

@ -18,9 +18,10 @@ endif()
if(NOT MSVC)
set (OPT_FLAGS "${OPT_FLAGS} -mavx2 -mbmi2 -mpclmul")
else()
set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2")
set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2 /std:c++latest")
endif()
if(NOT MSVC)
set(CXXSTD_FLAGS "-std=c++17 -fPIC")
endif()
@ -41,5 +42,5 @@ endif()
set(CMAKE_CXX_FLAGS "${CXXSTD_FLAGS} ${OPT_FLAGS} ${INCLUDE_FLAGS} ${WARNING_FLAGS} ${SIMDJSON_SANITIZE_FLAGS} ")
if(MSVC)
add_definitions( "/W3 /D_CRT_SECURE_NO_WARNINGS /wd4005 /wd4996 /wd4267 /wd4244 /wd4113 /nologo")
add_definitions( "${OPT_FLAGS} /W3 /D_CRT_SECURE_NO_WARNINGS /wd4005 /wd4996 /wd4267 /wd4244 /wd4113 /nologo")
endif()

14
tools/json2json.cpp
View File

@ -1,6 +1,7 @@
#include <iostream>
#ifndef _MSC_VER
#include <unistd.h>
#endif
#include "simdjson/jsonioutil.h"
#include "simdjson/jsonparser.h"
@ -41,9 +42,11 @@ void compute_dump(ParsedJson::iterator &pjh) {
}
int main(int argc, char *argv[]) {
bool rawdump = false;
bool apidump = false;
#ifndef _MSC_VER
int c;
bool rawdump = false;
bool apidump = false;
while ((c = getopt(argc, argv, "da")) != -1)
switch (c) {
@ -56,6 +59,9 @@ int main(int argc, char *argv[]) {
default:
abort();
}
#else
int optind = 1;
#endif
if (optind >= argc) {
cerr << "Reads json in, out the result of the parsing. " << endl;
cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;
@ -81,7 +87,7 @@ int main(int argc, char *argv[]) {
return EXIT_FAILURE;
}
bool is_ok = json_parse(p, pj); // do the parsing, return false on error
free((void *)p.data());
aligned_free((void *)p.data());
if (!is_ok) {
std::cerr << " Parsing failed. " << std::endl;
return EXIT_FAILURE;

10
tools/jsonstats.cpp
View File

@ -1,5 +1,4 @@
#include <iostream>
#include <unistd.h>
#include "simdjson/jsonioutil.h"
#include "simdjson/jsonparser.h"
@ -118,14 +117,7 @@ stat_t simdjson_computestats(const std::string_view &p) {
int main(int argc, char *argv[]) {
int c;
while ((c = getopt(argc, argv, "")) != -1)
switch (c) {
default:
abort();
}
int optind = 1;
if (optind >= argc) {
cerr << "Reads json, prints stats. " << endl;
cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;

2
tools/minify.cpp
View File

@ -18,5 +18,5 @@ int main(int argc, char *argv[]) {
}
jsonminify(p, (char *)p.data());
printf("%s",p.data());
free((void*)p.data());
aligned_free((void*)p.data());
}

1043
windows/dirent_portable.h Normal file
View File

File diff suppressed because it is too large Load Diff