jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Make "make amalgamate" more automatic (#480) 

- automatically include local includes in the right places
This commit is contained in:
John Keiser 2020-02-03 09:51:24 -08:00 committed by GitHub
parent c924aaede9
commit 0c8f2b9d85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 4826 additions and 4755 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
amalgamation.sh
View File

@ -11,69 +11,79 @@ echo "See https://www.sqlite.org/amalgamation.html and https://en.wikipedia.org/
AMAL_H="simdjson.h"
AMAL_C="simdjson.cpp"
SRCPATH="$SCRIPTPATH/src"
INCLUDEPATH="$SCRIPTPATH/include"
# this list excludes the "src/generic headers"
ALLCFILES="
$SCRIPTPATH/src/arm64/intrinsics.h
$SCRIPTPATH/src/haswell/intrinsics.h
$SCRIPTPATH/src/westmere/intrinsics.h
$SCRIPTPATH/src/simdprune_tables.h
$SCRIPTPATH/src/simdjson.cpp
$SCRIPTPATH/src/jsonioutil.cpp
$SCRIPTPATH/src/jsonminifier.cpp
$SCRIPTPATH/src/jsonparser.cpp
$SCRIPTPATH/src/arm64/bitmanipulation.h
$SCRIPTPATH/src/haswell/bitmanipulation.h
$SCRIPTPATH/src/westmere/bitmanipulation.h
$SCRIPTPATH/src/arm64/numberparsing.h
$SCRIPTPATH/src/haswell/numberparsing.h
$SCRIPTPATH/src/westmere/numberparsing.h
$SCRIPTPATH/src/arm64/bitmask.h
$SCRIPTPATH/src/haswell/bitmask.h
$SCRIPTPATH/src/westmere/bitmask.h
$SCRIPTPATH/src/arm64/simd.h
$SCRIPTPATH/src/haswell/simd.h
$SCRIPTPATH/src/westmere/simd.h
$SCRIPTPATH/src/arm64/stage1_find_marks.h
$SCRIPTPATH/src/haswell/stage1_find_marks.h
$SCRIPTPATH/src/westmere/stage1_find_marks.h
$SCRIPTPATH/src/stage1_find_marks.cpp
$SCRIPTPATH/src/arm64/stringparsing.h
$SCRIPTPATH/src/haswell/stringparsing.h
$SCRIPTPATH/src/westmere/stringparsing.h
$SCRIPTPATH/src/stage2_build_tape.cpp
$SCRIPTPATH/src/arm64/stage2_build_tape.h
$SCRIPTPATH/src/haswell/stage2_build_tape.h
$SCRIPTPATH/src/westmere/stage2_build_tape.h
$SCRIPTPATH/src/parsedjson.cpp
$SCRIPTPATH/src/parsedjsoniterator.cpp
simdjson.cpp
jsonioutil.cpp
jsonminifier.cpp
jsonparser.cpp
stage1_find_marks.cpp
stage2_build_tape.cpp
parsedjson.cpp
parsedjsoniterator.cpp
"
# order matters
ALLCHEADERS="
$SCRIPTPATH/include/simdjson/simdjson_version.h
$SCRIPTPATH/include/simdjson/portability.h
$SCRIPTPATH/include/simdjson/isadetection.h
$SCRIPTPATH/include/simdjson/jsonformatutils.h
$SCRIPTPATH/include/simdjson/simdjson.h
$SCRIPTPATH/include/simdjson/common_defs.h
$SCRIPTPATH/include/simdjson/padded_string.h
$SCRIPTPATH/include/simdjson/jsonioutil.h
$SCRIPTPATH/include/simdjson/jsonminifier.h
$SCRIPTPATH/include/simdjson/parsedjson.h
$SCRIPTPATH/include/simdjson/parsedjsoniterator.h
$SCRIPTPATH/include/simdjson/stage1_find_marks.h
$SCRIPTPATH/include/simdjson/stage2_build_tape.h
$SCRIPTPATH/include/simdjson/jsonparser.h
$SCRIPTPATH/src/jsoncharutils.h
$SCRIPTPATH/include/simdjson/jsonstream.h
simdjson/simdjson_version.h
simdjson/portability.h
simdjson/isadetection.h
simdjson/jsonformatutils.h
simdjson/simdjson.h
simdjson/common_defs.h
simdjson/padded_string.h
simdjson/jsonioutil.h
simdjson/jsonminifier.h
simdjson/parsedjson.h
simdjson/parsedjsoniterator.h
simdjson/stage1_find_marks.h
simdjson/stage2_build_tape.h
simdjson/jsonparser.h
simdjson/jsonstream.h
"
for i in ${ALLCHEADERS} ${ALLCFILES}; do
test -e $i && continue
echo "FATAL: source file [$i] not found."
found_includes=()
for file in ${ALLCFILES}; do
test -e "$SRCPATH/$file" && continue
echo "FATAL: source file [$SRCPATH/$file] not found."
exit 127
done
for file in ${ALLCHEADERS}; do
test -e "$INCLUDEPATH/$file" && continue
echo "FATAL: source file [$INCLUDEPATH/$file] not found."
exit 127
done
function doinclude()
{
file=$1
line="${@:2}"
if [ -f $INCLUDEPATH/$file ]; then
if [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
found_includes+=("$file")
dofile $INCLUDEPATH/$file
fi;
elif [ -f $SRCPATH/$file ]; then
# generic includes are included multiple times
if [[ "${file}" == *'generic/'*'.h' ]]; then
dofile $SRCPATH/$file
elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
found_includes+=("$file")
dofile $SRCPATH/$file
else
echo "/* $file already included: $line */"
fi
else
# If we don't recognize it, just emit the #include
echo "$line"
fi
}
function dofile()
{
# Last lines are always ignored. Files should end by an empty lines.
@ -86,23 +96,15 @@ function dofile()
file=$(echo $line| cut -d'"' -f 2)
if [[ "${file}" == '../'* ]]; then
file=$(echo $file| cut -d'/' -f 2-)
file=$(echo $file| cut -d'/' -f 2-)
fi;
# we ignore simdjson headers (except src/generic/*.h); they are handled in the above list
if [ -f include/$file ]; then
continue;
elif [ -f src/$file ]; then
# we paste the contents of src/generic/*.h
if [[ "${file}" == *'generic/'*'.h' ]]; then
echo "$(<src/$file)"
fi;
continue;
fi;
fi;
# Otherwise we simply copy the line
echo "$line"
# we explicitly include simdjson headers, one time each (unless they are generic, in which case multiple times is fine)
doinclude $file $line
else
# Otherwise we simply copy the line
echo "$line"
fi
done < "$1"
echo "/* end file $RELFILE */"
}
@ -111,7 +113,7 @@ echo "Creating ${AMAL_H}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_H}"
{
for h in ${ALLCHEADERS}; do
dofile $h
doinclude $h "ERROR $h not found"
done
} >> "${AMAL_H}"
@ -128,13 +130,12 @@ echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
echo "#endif"
echo ""
for h in ${ALLCFILES}; do
dofile $h
for file in ${ALLCFILES}; do
dofile "$SRCPATH/$file"
done
} >> "${AMAL_C}"
DEMOCPP="amalgamation_demo.cpp"
echo "Creating ${DEMOCPP}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${DEMOCPP}"

2
singleheader/amalgamation_demo.cpp
View File

@ -1,4 +1,4 @@
/* auto-generated on Thu Jan 30 10:52:58 EST 2020. Do not edit! */
/* auto-generated on Sun Feb 2 15:10:09 PST 2020. Do not edit! */
#include <iostream>
#include "simdjson.h"

9410
singleheader/simdjson.cpp
View File

File diff suppressed because it is too large Load Diff

26
singleheader/simdjson.h
View File

@ -1,4 +1,4 @@
/* auto-generated on Thu Jan 30 10:52:58 EST 2020. Do not edit! */
/* auto-generated on Sun Feb 2 15:10:09 PST 2020. Do not edit! */
/* begin file include/simdjson/simdjson_version.h */
// /include/simdjson/simdjson_version.h automatically generated by release.py,
// do not change by hand
@ -1904,6 +1904,14 @@ inline ParsedJson build_parsed_json(const padded_string &s) {
} // namespace simdjson
#endif
/* end file include/simdjson/jsonparser.h */
/* begin file include/simdjson/jsonstream.h */
#ifndef SIMDJSON_JSONSTREAM_H
#define SIMDJSON_JSONSTREAM_H
#include <algorithm>
#include <limits>
#include <stdexcept>
#include <thread>
/* begin file src/jsoncharutils.h */
#ifndef SIMDJSON_JSONCHARUTILS_H
#define SIMDJSON_JSONCHARUTILS_H
@ -2229,12 +2237,6 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
#endif
/* end file src/jsoncharutils.h */
/* begin file include/simdjson/jsonstream.h */
#ifndef SIMDJSON_JSONSTREAM_H
#define SIMDJSON_JSONSTREAM_H
#include <stdexcept>
#include <thread>
namespace simdjson {
@ -2358,7 +2360,7 @@ private:
inline size_t remaining() const { return str.size() - str_start; }
const string_container &str;
size_t _batch_size;
size_t _batch_size; // this is actually variable!
size_t str_start{0};
size_t next_json{0};
bool load_next_batch{true};
@ -2534,7 +2536,7 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
if (unlikely(load_next_batch)) {
// First time loading
if (!stage_1_thread.joinable()) {
_batch_size = std::min(_batch_size, remaining());
_batch_size = (std::min)(_batch_size, remaining());
_batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size);
if (_batch_size == 0) {
pj.error_code = simdjson::UTF8_ERROR;
@ -2571,7 +2573,7 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
if (remaining() - _batch_size > 0) {
last_json_buffer_loc =
pj.structural_indexes[find_last_json_buf_idx(buf(), _batch_size, pj)];
_batch_size = std::min(_batch_size, remaining() - last_json_buffer_loc);
_batch_size = (std::min)(_batch_size, remaining() - last_json_buffer_loc);
if (_batch_size > 0) {
_batch_size = trimmed_length_safe_utf8(
(const char *)(buf() + last_json_buffer_loc), _batch_size);
@ -2627,7 +2629,7 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
if (unlikely(load_next_batch)) {
advance(current_buffer_loc);
n_bytes_parsed += current_buffer_loc;
_batch_size = std::min(_batch_size, remaining());
_batch_size = (std::min)(_batch_size, remaining());
_batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size);
int stage1_is_ok = best_stage1(buf(), _batch_size, pj, true);
if (stage1_is_ok != simdjson::SUCCESS) {
@ -2664,4 +2666,4 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
} // end of namespace simdjson
#endif // SIMDJSON_JSONSTREAM_H
/* end file include/simdjson/jsonstream.h */
/* end file src/jsoncharutils.h */