From a00df9b99291f5fd5bd72f16b4924388a62536e1 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Fri, 4 Jan 2019 17:36:52 -0500 Subject: [PATCH] Number parsing fix. --- include/simdjson/numberparsing.h | 22 +++++++++++++++- singleheader/amalgamation_demo.cpp | 2 +- singleheader/simdjson.cpp | 8 +++--- singleheader/simdjson.h | 40 ++++++++++++++++++++++-------- 4 files changed, 56 insertions(+), 16 deletions(-) diff --git a/include/simdjson/numberparsing.h b/include/simdjson/numberparsing.h index 61fb6c68..baff3416 100644 --- a/include/simdjson/numberparsing.h +++ b/include/simdjson/numberparsing.h @@ -189,6 +189,17 @@ parse_float(const uint8_t *const buf, if ('.' == *p) { ++p; double fractionalweight = 1; + if(is_integer(*p)) { + unsigned char digit = *p - '0'; + ++p; + fractionalweight *= 0.1; + i = i + digit * fractionalweight; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + foundInvalidNumber(buf + offset); +#endif + return false; + } while (is_integer(*p)) { unsigned char digit = *p - '0'; ++p; @@ -389,7 +400,16 @@ static really_inline bool parse_number(const uint8_t *const buf, if ('.' == *p) { ++p; const char *const firstafterperiod = p; - + if(is_integer(*p)) { + unsigned char digit = *p - '0'; + ++p; + i = i * 10 + digit; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + foundInvalidNumber(buf + offset); +#endif + return false; + } #ifdef SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index 880f5dcc..973cf963 100644 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Mon Dec 31 17:13:28 EST 2018. Do not edit! */ +/* auto-generated on Fri Jan 4 17:36:46 EST 2019. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 9a720cb4..bfec0701 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Mon Dec 31 17:13:28 EST 2018. Do not edit! */ +/* auto-generated on Fri Jan 4 17:36:46 EST 2019. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ @@ -814,8 +814,8 @@ WARN_UNUSED return true; #endif } -/* end file /Users/lemire/CVS/github/simdjson/src/stage1_find_marks.cpp */ -/* begin file /Users/lemire/CVS/github/simdjson/src/stage2_build_tape.cpp */ +/* end file /home/dlemire/CVS/github/simdjson/src/stage1_find_marks.cpp */ +/* begin file /home/dlemire/CVS/github/simdjson/src/stage2_build_tape.cpp */ #ifdef _MSC_VER /* Microsoft C/C++-compatible compiler */ #include @@ -1322,4 +1322,4 @@ succeed: fail: return false; } -/* end file /Users/lemire/CVS/github/simdjson/src/stage2_build_tape.cpp */ +/* end file /home/dlemire/CVS/github/simdjson/src/stage2_build_tape.cpp */ diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 0bbccd84..d440a4be 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,5 +1,5 @@ -/* auto-generated on Mon Dec 31 17:13:28 EST 2018. Do not edit! */ -/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/portability.h */ +/* auto-generated on Fri Jan 4 17:36:46 EST 2019. Do not edit! */ +/* begin file /home/dlemire/CVS/github/simdjson/include/simdjson/portability.h */ #ifndef SIMDJSON_PORTABILITY_H #define SIMDJSON_PORTABILITY_H @@ -36515,8 +36515,8 @@ static inline bool find_structural_bits(const char *buf, size_t len, ParsedJson } #endif -/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/stage1_find_marks.h */ -/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/stringparsing.h */ +/* end file /home/dlemire/CVS/github/simdjson/include/simdjson/stage1_find_marks.h */ +/* begin file /home/dlemire/CVS/github/simdjson/include/simdjson/stringparsing.h */ #ifndef SIMDJSON_STRINGPARSING_H #define SIMDJSON_STRINGPARSING_H @@ -36627,7 +36627,7 @@ really_inline bool parse_string(const uint8_t *buf, UNUSED size_t len, return is_ok; #else //CHECKUNESCAPED #ifdef JSON_TEST_STRINGS // for unit testing - foundString(buf + offset,start_of_string,pj.current_string_buf_loc); + foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1); #endif // JSON_TEST_STRINGS return true; #endif //CHECKUNESCAPED @@ -36883,6 +36883,17 @@ parse_float(const uint8_t *const buf, if ('.' == *p) { ++p; double fractionalweight = 1; + if(is_integer(*p)) { + unsigned char digit = *p - '0'; + ++p; + fractionalweight *= 0.1; + i = i + digit * fractionalweight; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + foundInvalidNumber(buf + offset); +#endif + return false; + } while (is_integer(*p)) { unsigned char digit = *p - '0'; ++p; @@ -37083,7 +37094,16 @@ static really_inline bool parse_number(const uint8_t *const buf, if ('.' == *p) { ++p; const char *const firstafterperiod = p; - + if(is_integer(*p)) { + unsigned char digit = *p - '0'; + ++p; + i = i * 10 + digit; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + foundInvalidNumber(buf + offset); +#endif + return false; + } #ifdef SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. @@ -37192,8 +37212,8 @@ static really_inline bool parse_number(const uint8_t *const buf, } #endif -/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/numberparsing.h */ -/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/stage2_build_tape.h */ +/* end file /home/dlemire/CVS/github/simdjson/include/simdjson/numberparsing.h */ +/* begin file /home/dlemire/CVS/github/simdjson/include/simdjson/stage2_build_tape.h */ #ifndef SIMDJSON_STAGE34_UNIFIED_H #define SIMDJSON_STAGE34_UNIFIED_H @@ -37209,8 +37229,8 @@ static inline bool unified_machine(const char *buf, size_t len, ParsedJson &pj) } #endif -/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/stage2_build_tape.h */ -/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonparser.h */ +/* end file /home/dlemire/CVS/github/simdjson/include/simdjson/stage2_build_tape.h */ +/* begin file /home/dlemire/CVS/github/simdjson/include/simdjson/jsonparser.h */ #ifndef SIMDJSON_JSONPARSER_H #define SIMDJSON_JSONPARSER_H