Make "make amalgamate" more automatic (#480)

- automatically include local includes in the right places

amalgamation.sh

@@ -11,69 +11,79 @@ echo "See https://www.sqlite.org/amalgamation.html and https://en.wikipedia.org/
 AMAL_H="simdjson.h"
 AMAL_C="simdjson.cpp"
 
+SRCPATH="$SCRIPTPATH/src"
+INCLUDEPATH="$SCRIPTPATH/include"
+
 # this list excludes the "src/generic headers"
 ALLCFILES="
-$SCRIPTPATH/src/arm64/intrinsics.h
+simdjson.cpp
-$SCRIPTPATH/src/haswell/intrinsics.h
+jsonioutil.cpp
-$SCRIPTPATH/src/westmere/intrinsics.h
+jsonminifier.cpp
-$SCRIPTPATH/src/simdprune_tables.h
+jsonparser.cpp
-$SCRIPTPATH/src/simdjson.cpp
+stage1_find_marks.cpp
-$SCRIPTPATH/src/jsonioutil.cpp
+stage2_build_tape.cpp
-$SCRIPTPATH/src/jsonminifier.cpp
+parsedjson.cpp
-$SCRIPTPATH/src/jsonparser.cpp
+parsedjsoniterator.cpp
-$SCRIPTPATH/src/arm64/bitmanipulation.h
-$SCRIPTPATH/src/haswell/bitmanipulation.h
-$SCRIPTPATH/src/westmere/bitmanipulation.h
-$SCRIPTPATH/src/arm64/numberparsing.h
-$SCRIPTPATH/src/haswell/numberparsing.h
-$SCRIPTPATH/src/westmere/numberparsing.h
-$SCRIPTPATH/src/arm64/bitmask.h
-$SCRIPTPATH/src/haswell/bitmask.h
-$SCRIPTPATH/src/westmere/bitmask.h
-$SCRIPTPATH/src/arm64/simd.h
-$SCRIPTPATH/src/haswell/simd.h
-$SCRIPTPATH/src/westmere/simd.h
-$SCRIPTPATH/src/arm64/stage1_find_marks.h
-$SCRIPTPATH/src/haswell/stage1_find_marks.h
-$SCRIPTPATH/src/westmere/stage1_find_marks.h
-$SCRIPTPATH/src/stage1_find_marks.cpp
-$SCRIPTPATH/src/arm64/stringparsing.h
-$SCRIPTPATH/src/haswell/stringparsing.h
-$SCRIPTPATH/src/westmere/stringparsing.h
-$SCRIPTPATH/src/stage2_build_tape.cpp
-$SCRIPTPATH/src/arm64/stage2_build_tape.h
-$SCRIPTPATH/src/haswell/stage2_build_tape.h
-$SCRIPTPATH/src/westmere/stage2_build_tape.h
-$SCRIPTPATH/src/parsedjson.cpp
-$SCRIPTPATH/src/parsedjsoniterator.cpp
 "
 
 # order matters
 ALLCHEADERS="
-$SCRIPTPATH/include/simdjson/simdjson_version.h
+simdjson/simdjson_version.h
-$SCRIPTPATH/include/simdjson/portability.h
+simdjson/portability.h
-$SCRIPTPATH/include/simdjson/isadetection.h
+simdjson/isadetection.h
-$SCRIPTPATH/include/simdjson/jsonformatutils.h
+simdjson/jsonformatutils.h
-$SCRIPTPATH/include/simdjson/simdjson.h
+simdjson/simdjson.h
-$SCRIPTPATH/include/simdjson/common_defs.h
+simdjson/common_defs.h
-$SCRIPTPATH/include/simdjson/padded_string.h
+simdjson/padded_string.h
-$SCRIPTPATH/include/simdjson/jsonioutil.h
+simdjson/jsonioutil.h
-$SCRIPTPATH/include/simdjson/jsonminifier.h
+simdjson/jsonminifier.h
-$SCRIPTPATH/include/simdjson/parsedjson.h
+simdjson/parsedjson.h
-$SCRIPTPATH/include/simdjson/parsedjsoniterator.h
+simdjson/parsedjsoniterator.h
-$SCRIPTPATH/include/simdjson/stage1_find_marks.h
+simdjson/stage1_find_marks.h
-$SCRIPTPATH/include/simdjson/stage2_build_tape.h
+simdjson/stage2_build_tape.h
-$SCRIPTPATH/include/simdjson/jsonparser.h
+simdjson/jsonparser.h
-$SCRIPTPATH/src/jsoncharutils.h
+simdjson/jsonstream.h
-$SCRIPTPATH/include/simdjson/jsonstream.h
 "
 
-for i in ${ALLCHEADERS} ${ALLCFILES}; do
+found_includes=()
-    test -e $i && continue
+
-    echo "FATAL: source file [$i] not found."
+for file in ${ALLCFILES}; do
+    test -e "$SRCPATH/$file" && continue
+    echo "FATAL: source file [$SRCPATH/$file] not found."
     exit 127
 done
 
+for file in ${ALLCHEADERS}; do
+    test -e "$INCLUDEPATH/$file" && continue
+    echo "FATAL: source file [$INCLUDEPATH/$file] not found."
+    exit 127
+done
+
+function doinclude()
+{
+    file=$1
+    line="${@:2}"
+    if [ -f $INCLUDEPATH/$file ]; then
+        if [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
+            found_includes+=("$file")
+            dofile $INCLUDEPATH/$file
+        fi;
+    elif [ -f $SRCPATH/$file ]; then
+        # generic includes are included multiple times
+        if [[ "${file}" == *'generic/'*'.h' ]]; then
+            dofile $SRCPATH/$file
+        elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
+            found_includes+=("$file")
+            dofile $SRCPATH/$file
+        else
+            echo "/* $file already included: $line */"
+        fi
+    else
+      # If we don't recognize it, just emit the #include
+      echo "$line"
+    fi
+}
+
 function dofile()
 {
     # Last lines are always ignored. Files should end by an empty lines.
@@ -89,20 +99,12 @@ function dofile()
                 file=$(echo $file| cut -d'/' -f 2-)
             fi;
 
-            # we ignore simdjson headers (except src/generic/*.h); they are handled in the above list
+            # we explicitly include simdjson headers, one time each (unless they are generic, in which case multiple times is fine)
-            if [ -f include/$file ]; then
+            doinclude $file $line
-              continue;
+        else
-            elif [ -f src/$file ]; then
-              # we paste the contents of src/generic/*.h
-              if [[ "${file}" == *'generic/'*'.h' ]]; then
-                echo "$(<src/$file)"
-              fi;
-              continue;
-            fi;
-        fi;
-
             # Otherwise we simply copy the line
             echo "$line"
+        fi
     done < "$1"
     echo "/* end file $RELFILE */"
 }
@@ -111,7 +113,7 @@ echo "Creating ${AMAL_H}..."
 echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_H}"
 {
     for h in ${ALLCHEADERS}; do
-        dofile $h
+        doinclude $h "ERROR $h not found"
     done
 } >> "${AMAL_H}"
 
@@ -128,13 +130,12 @@ echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
     echo "#endif"
     echo ""
 
-    for h in ${ALLCFILES}; do
+    for file in ${ALLCFILES}; do
-        dofile $h
+        dofile "$SRCPATH/$file"
     done
 } >> "${AMAL_C}"
 
 
-
 DEMOCPP="amalgamation_demo.cpp"
 echo "Creating ${DEMOCPP}..."
 echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${DEMOCPP}"

singleheader/amalgamation_demo.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on Thu Jan 30 10:52:58 EST 2020. Do not edit! */
+/* auto-generated on Sun Feb  2 15:10:09 PST 2020. Do not edit! */
 
 #include <iostream>
 #include "simdjson.h"

singleheader/simdjson.h

@@ -1,4 +1,4 @@
-/* auto-generated on Thu Jan 30 10:52:58 EST 2020. Do not edit! */
+/* auto-generated on Sun Feb  2 15:10:09 PST 2020. Do not edit! */
 /* begin file include/simdjson/simdjson_version.h */
 // /include/simdjson/simdjson_version.h automatically generated by release.py,
 // do not change by hand
@@ -1904,6 +1904,14 @@ inline ParsedJson build_parsed_json(const padded_string &s) {
 } // namespace simdjson
 #endif
 /* end file include/simdjson/jsonparser.h */
+/* begin file include/simdjson/jsonstream.h */
+#ifndef SIMDJSON_JSONSTREAM_H
+#define SIMDJSON_JSONSTREAM_H
+
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+#include <thread>
 /* begin file src/jsoncharutils.h */
 #ifndef SIMDJSON_JSONCHARUTILS_H
 #define SIMDJSON_JSONCHARUTILS_H
@@ -2229,12 +2237,6 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
 
 #endif
 /* end file src/jsoncharutils.h */
-/* begin file include/simdjson/jsonstream.h */
-#ifndef SIMDJSON_JSONSTREAM_H
-#define SIMDJSON_JSONSTREAM_H
-
-#include <stdexcept>
-#include <thread>
 
 
 namespace simdjson {
@@ -2358,7 +2360,7 @@ private:
   inline size_t remaining() const { return str.size() - str_start; }
 
   const string_container &str;
-  size_t _batch_size;
+  size_t _batch_size; // this is actually variable!
   size_t str_start{0};
   size_t next_json{0};
   bool load_next_batch{true};
@@ -2534,7 +2536,7 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
   if (unlikely(load_next_batch)) {
     // First time loading
     if (!stage_1_thread.joinable()) {
-      _batch_size = std::min(_batch_size, remaining());
+      _batch_size = (std::min)(_batch_size, remaining());
       _batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size);
       if (_batch_size == 0) {
         pj.error_code = simdjson::UTF8_ERROR;
@@ -2571,7 +2573,7 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
     if (remaining() - _batch_size > 0) {
       last_json_buffer_loc =
           pj.structural_indexes[find_last_json_buf_idx(buf(), _batch_size, pj)];
-      _batch_size = std::min(_batch_size, remaining() - last_json_buffer_loc);
+      _batch_size = (std::min)(_batch_size, remaining() - last_json_buffer_loc);
       if (_batch_size > 0) {
         _batch_size = trimmed_length_safe_utf8(
             (const char *)(buf() + last_json_buffer_loc), _batch_size);
@@ -2627,7 +2629,7 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
   if (unlikely(load_next_batch)) {
     advance(current_buffer_loc);
     n_bytes_parsed += current_buffer_loc;
-    _batch_size = std::min(_batch_size, remaining());
+    _batch_size = (std::min)(_batch_size, remaining());
     _batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size);
     int stage1_is_ok = best_stage1(buf(), _batch_size, pj, true);
     if (stage1_is_ok != simdjson::SUCCESS) {
@@ -2664,4 +2666,4 @@ int JsonStream<string_container>::json_parse(ParsedJson &pj) {
 
 } // end of namespace simdjson
 #endif // SIMDJSON_JSONSTREAM_H
-/* end file include/simdjson/jsonstream.h */
+/* end file src/jsoncharutils.h */