jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Store buf and len in parser implementation

This commit is contained in:
John Keiser 2020-06-01 12:14:09 -07:00
parent b86fb95306
commit 84712a8bbc
10 changed files with 57 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/arm64/dom_parser_implementation.cpp
View File

@ -84,8 +84,10 @@ WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, ui
#include "generic/stage1/utf8_lookup2_algorithm.h"
#include "generic/stage1/json_structural_indexer.h"
WARN_UNUSED error_code implementation::stage1(const uint8_t *buf, size_t len, parser &parser, bool streaming) const noexcept {
return arm64::stage1::json_structural_indexer::index<64>(buf, len, parser, streaming);
WARN_UNUSED error_code implementation::stage1(const uint8_t *_buf, size_t _len, parser &parser, bool streaming) const noexcept {
this->buf = _buf;
this->len = _len;
return arm64::stage1::json_structural_indexer::index<64>(_buf, _len, parser, streaming);
}
} // namespace arm64

11
src/arm64/dom_parser_implementation.h
View File

@ -9,7 +9,12 @@ namespace arm64 {
class dom_parser_implementation final : public internal::dom_parser_implementation {
public:
const uint8_t *buf{}; // Buffer passed to stage 1
size_t len{0}; // Length passed to stage 1
really_inline dom_parser_implementation();
dom_parser_implementation(const dom_parser_implementation &) = delete;
dom_parser_implementation &operator =(const dom_parser_implementation &) = delete;
WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept;
WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, dom::parser &parser, bool streaming) noexcept;
@ -30,10 +35,10 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t) noexcept
return SUCCESS;
}
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept {
error_code code = stage1(buf, len, parser, false);
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::parser &parser) noexcept {
error_code code = stage1(_buf, _len, parser, false);
if (!code) {
code = stage2(buf, len, parser);
code = stage2(_buf, _len, parser);
}
return code;
}

8
src/fallback/dom_parser_implementation.cpp
View File

@ -145,11 +145,13 @@ private:
} // namespace stage1
WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *buf, size_t len, dom::parser &parser, bool streaming) noexcept {
if (unlikely(len > parser.capacity())) {
WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, dom::parser &parser, bool streaming) noexcept {
if (unlikely(_len > parser.capacity())) {
return CAPACITY;
}
stage1::structural_scanner scanner(buf, uint32_t(len), parser, streaming);
this->buf = _buf;
this->len = _len;
stage1::structural_scanner scanner(_buf, uint32_t(_len), parser, streaming);
return scanner.scan();
}

11
src/fallback/dom_parser_implementation.h
View File

@ -9,7 +9,12 @@ namespace fallback {
class dom_parser_implementation final : public internal::dom_parser_implementation {
public:
const uint8_t *buf{}; // Buffer passed to stage 1
size_t len{0}; // Length passed to stage 1
really_inline dom_parser_implementation();
dom_parser_implementation(const dom_parser_implementation &) = delete;
dom_parser_implementation &operator =(const dom_parser_implementation &) = delete;
WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept;
WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, dom::parser &parser, bool streaming) noexcept;
@ -30,10 +35,10 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t) noexcept
return SUCCESS;
}
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept {
error_code code = stage1(buf, len, parser, false);
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::parser &parser) noexcept {
error_code code = stage1(_buf, _len, parser, false);
if (!code) {
code = stage2(buf, len, parser);
code = stage2(_buf, _len, parser);
}
return code;
}

9
src/generic/stage2/streaming_structural_parser.h
View File

@ -4,7 +4,7 @@ struct streaming_structural_parser: structural_parser {
really_inline streaming_structural_parser(const uint8_t *buf, size_t len, dom::parser &_doc_parser, uint32_t next_structural) : structural_parser(buf, len, _doc_parser, next_structural) {}
// override to add streaming
WARN_UNUSED really_inline error_code start(UNUSED size_t len, ret_address finish_parser) {
WARN_UNUSED really_inline error_code start(ret_address finish_parser) {
log_start();
init(); // sets is_valid to false
// Capacity ain't no thang for streaming, so we don't check it.
@ -44,10 +44,11 @@ struct streaming_structural_parser: structural_parser {
* The JSON is parsed to a tape, see the accompanying tape.md file
* for documentation.
***********/
WARN_UNUSED error_code dom_parser_implementation::stage2(const uint8_t *buf, size_t len, dom::parser &doc_parser, size_t &next_json) noexcept {
WARN_UNUSED error_code dom_parser_implementation::stage2(const uint8_t *_buf, size_t _len, dom::parser &doc_parser, size_t &next_json) noexcept {
static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES();
stage2::streaming_structural_parser parser(buf, len, doc_parser, uint32_t(next_json));
error_code result = parser.start(len, addresses.finish);
stage2::streaming_structural_parser parser(_buf, _len, doc_parser, uint32_t(next_json));
error_code result = parser.start(addresses.finish);
if (result) { return result; }
//
// Read first value

6
src/generic/stage2/structural_parser.h
View File

@ -398,10 +398,10 @@ struct structural_parser {
* The JSON is parsed to a tape, see the accompanying tape.md file
* for documentation.
***********/
WARN_UNUSED error_code dom_parser_implementation::stage2(const uint8_t *buf, size_t len, dom::parser &doc_parser) noexcept {
WARN_UNUSED error_code dom_parser_implementation::stage2(const uint8_t *_buf, size_t _len, dom::parser &doc_parser) noexcept {
static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES();
stage2::structural_parser parser(buf, len, doc_parser);
error_code result = parser.start(len, addresses.finish);
stage2::structural_parser parser(_buf, _len, doc_parser);
error_code result = parser.start(_len, addresses.finish);
if (result) { return result; }
//

6
src/haswell/dom_parser_implementation.cpp
View File

@ -72,8 +72,10 @@ WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, ui
#include "generic/stage1/utf8_lookup2_algorithm.h"
#include "generic/stage1/json_structural_indexer.h"
WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *buf, size_t len, dom::parser &parser, bool streaming) noexcept {
return haswell::stage1::json_structural_indexer::index<128>(buf, len, parser, streaming);
WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, dom::parser &parser, bool streaming) noexcept {
this->buf = _buf;
this->len = _len;
return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, parser, streaming);
}
} // namespace haswell

11
src/haswell/dom_parser_implementation.h
View File

@ -9,7 +9,12 @@ namespace haswell {
class dom_parser_implementation final : public internal::dom_parser_implementation {
public:
const uint8_t *buf{}; // Buffer passed to stage 1
size_t len{0}; // Length passed to stage 1
really_inline dom_parser_implementation();
dom_parser_implementation(const dom_parser_implementation &) = delete;
dom_parser_implementation &operator =(const dom_parser_implementation &) = delete;
WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept;
WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, dom::parser &parser, bool streaming) noexcept;
@ -30,10 +35,10 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t) noexcept
return SUCCESS;
}
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept {
error_code code = stage1(buf, len, parser, false);
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::parser &parser) noexcept {
error_code code = stage1(_buf, _len, parser, false);
if (!code) {
code = stage2(buf, len, parser);
code = stage2(_buf, _len, parser);
}
return code;
}

6
src/westmere/dom_parser_implementation.cpp
View File

@ -73,8 +73,10 @@ WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, ui
#include "generic/stage1/utf8_lookup2_algorithm.h"
#include "generic/stage1/json_structural_indexer.h"
WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *buf, size_t len, parser &parser, bool streaming) noexcept {
return westmere::stage1::json_structural_indexer::index<64>(buf, len, parser, streaming);
WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, parser &parser, bool streaming) noexcept {
this->buf = _buf;
this->len = _len;
return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, parser, streaming);
}
} // namespace westmere

11
src/westmere/dom_parser_implementation.h
View File

@ -9,7 +9,12 @@ namespace westmere {
class dom_parser_implementation final : public internal::dom_parser_implementation {
public:
const uint8_t *buf{}; // Buffer passed to stage 1
size_t len{0}; // Length passed to stage 1
really_inline dom_parser_implementation();
dom_parser_implementation(const dom_parser_implementation &) = delete;
dom_parser_implementation &operator =(const dom_parser_implementation &) = delete;
WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept;
WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, dom::parser &parser, bool streaming) noexcept;
@ -30,10 +35,10 @@ WARN_UNUSED error_code dom_parser_implementation::set_max_depth(size_t) noexcept
return SUCCESS;
}
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *buf, size_t len, dom::parser &parser) noexcept {
error_code code = stage1(buf, len, parser, false);
WARN_UNUSED error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::parser &parser) noexcept {
error_code code = stage1(_buf, _len, parser, false);
if (!code) {
code = stage2(buf, len, parser);
code = stage2(_buf, _len, parser);
}
return code;
}