Stupid work.

README.md

@@ -59,7 +59,7 @@ To simplify the engineering, we make some assumptions.
 - We assume AVX2 support which is available in all recent mainstream x86 processors produced by AMD and Intel. No support for non-x86 processors is included.
 - We only support GNU GCC and LLVM Clang at this time. There is no support for Microsoft Visual Studio, though it should not be difficult.
 - We expect the input memory pointer to be padded (e.g., with spaces) so that it can be read entirely in blocks of 512 bits (a cache line). In practice, this means that users should allocate the memory where the JSON bytes are located using the `allocate_aligned_buffer` function or the equivalent. Of course, the data you may want to processed could be on a buffer that does have this padding. However, copying the data is relatively cheap (much cheaper than parsing JSON), and we can eventually remove this constraint.
-
+- The input string should be NULL terminated.
 
 ## Features
 

include/jsonparser/jsonparser.h

@@ -19,4 +19,5 @@ void deallocate_ParsedJson(ParsedJson *pj_ptr);
 
 // Parse a document found in buf, need to preallocate ParsedJson.
 // Return false in case of a failure.
+// The string should be NULL terminated.
 bool json_parse(const u8 *buf, size_t len, ParsedJson &pj);

include/jsonparser/simdjson_internal.h

@@ -12,13 +12,13 @@
 #include <iomanip>
 
 
-const u32 MAX_DEPTH = 256;
-const u32 DEPTH_SAFETY_MARGIN = 32; // should be power-of-2 as we check this
+//const u32 MAX_DEPTH = 2048;
+//const u32 DEPTH_SAFETY_MARGIN = 32; // should be power-of-2 as we check this
                                     // with a modulo in our hot stage 3 loop
-const u32 START_DEPTH = DEPTH_SAFETY_MARGIN;
-const u32 REDLINE_DEPTH = MAX_DEPTH - DEPTH_SAFETY_MARGIN;
-const size_t MAX_TAPE_ENTRIES = 127 * 1024;
-const size_t MAX_TAPE = MAX_DEPTH * MAX_TAPE_ENTRIES;
+//const u32 START_DEPTH = DEPTH_SAFETY_MARGIN;
+//const u32 REDLINE_DEPTH = MAX_DEPTH - DEPTH_SAFETY_MARGIN;
+//const size_t MAX_TAPE_ENTRIES = 127 * 1024;
+//const size_t MAX_TAPE = MAX_DEPTH * MAX_TAPE_ENTRIES;
 
 /////////////
 // TODO: move this to be more like a real class
@@ -31,13 +31,13 @@ struct ParsedJson {
 public:
   size_t bytecapacity; // indicates how many bits are meant to be supported by
                        // structurals
+  size_t depthcapacity; // how deep we can go
   u8 *structurals;
   u32 n_structural_indexes;
   u32 *structural_indexes;
 
-  // grossly overprovisioned
-  u64 tape[MAX_TAPE];
-  u32 tape_locs[MAX_DEPTH];
+  u64 * tape;//[MAX_TAPE];
+  u32 * tape_locs;
   u8 * string_buf;// should be at least bytecapacity
   u8 *current_string_buf_loc;
   u8 * number_buf;// holds either doubles or longs, really // should be at least 4 * bytecapacity
@@ -47,13 +47,14 @@ public:
         current_string_buf_loc = string_buf;
         current_number_buf_loc = number_buf;
 
-        for (u32 i = 0; i < MAX_DEPTH; i++) {
-            tape_locs[i] = i * MAX_TAPE_ENTRIES;
-        }
+        //for (u32 i = 0; i < MAX_DEPTH; i++) {
+        //    tape_locs[i] = i * MAX_TAPE_ENTRIES;
+        //}
+        //tap_locs will be unitialized by design
     }
 
     void dump_tapes() {
-        for (u32 i = 0; i < MAX_DEPTH; i++) {
+        /*for (u32 i = 0; i < MAX_DEPTH; i++) {
             u32 start_loc = i * MAX_TAPE_ENTRIES;
             std::cout << " tape section i " << i;
             if (i == START_DEPTH) {
@@ -72,7 +73,7 @@ public:
                               << " tape[j][0..55]: " << (tape[j] & 0xffffffffffffffULL) << "\n";
                 }
             }
-        }
+        }*/
     }
     // TODO: will need a way of saving strings that's a bit more encapsulated
 

src/jsonparser.cpp

@@ -30,31 +30,34 @@ ParsedJson *allocate_ParsedJson(size_t len) {
     return NULL;
   }
   pj.string_buf = new u8[ROUNDUP_N(len, 64)];
-  if (pj.string_buf == NULL) {
-    std::cerr << "Could not allocate memory for string_buf"
-              << std::endl;
-    delete[] pj.structural_indexes;
-    delete[] pj.structurals;
-    delete pj_ptr;
-    return NULL;
-  }
   pj.number_buf = new u8[4 * ROUNDUP_N(len, 64)];
-  if (pj.string_buf == NULL) {
-    std::cerr << "Could not allocate memory for number_buf"
+  pj.tape = new u64[ROUNDUP_N(len, 64)];
+  size_t depthcapacity = ROUNDUP_N(len, 64);
+  pj.tape_locs = new u32[depthcapacity];
+
+  if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL) || (pj.tape_locs == NULL))  {
+    std::cerr << "Could not allocate memory"
               << std::endl;
+    delete[] pj.tape_locs;
+    delete[] pj.tape;
+    delete[] pj.number_buf;
     delete[] pj.string_buf;
     delete[] pj.structural_indexes;
     delete[] pj.structurals;
     delete pj_ptr;
     return NULL;
   }
+
   pj.bytecapacity = len;
+  pj.depthcapacity =  depthcapacity;
   return pj_ptr;
 }
 
 void deallocate_ParsedJson(ParsedJson *pj_ptr) {
   if (pj_ptr == NULL)
     return;
+  delete[] pj_ptr->tape_locs;
+  delete[] pj_ptr->tape;
   delete[] pj_ptr->number_buf;
   delete[] pj_ptr->string_buf;
   delete[] pj_ptr->structural_indexes;

src/stage1_find_marks.cpp

@@ -250,6 +250,12 @@ WARN_UNUSED
         "final structurals and pseudo structurals after close quote removal");
     *(u64 *)(pj.structurals + idx / 8) = structurals;
   }
+  if(buf[len] != '\0') {
+      std::cerr << "Your string should NULL terminated." << std::endl;
+      return false;
+
+  }
+  pj.structural_indexes[pj.n_structural_indexes++] = len; // the final NULL is used as a pseudo-structural character
 #ifdef UTF8VALIDATE
   return _mm256_testz_si256(has_error, has_error);
 #else

src/stage34_unified.cpp

@@ -67,7 +67,7 @@ really_inline bool is_valid_null_atom(const u8 * loc) {
     return error == 0;
 }
 
-// Implemented using Labels as Values which works in GCC and CLANG,
+// Implemented using Labels as Values which works in GCC and CLANG (and maybe also in Intel's compiler),
 // but won't work in MSVC. This would need to be reimplemented differently
 // if one wants to be standard compliant.
 WARN_UNUSED
@@ -75,11 +75,11 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
     u32 i = 0; // index of the structural character (0,1,2,3...)
     u32 idx; // location of the structural character in the input (buf)
     u8 c; // used to track the (structural) character we are looking at, updated by UPDATE_CHAR macro
-    u32 depth = START_DEPTH; // an arbitrary starting depth
-    void * ret_address[MAX_DEPTH]; // used to store "labels as value" (non-standard compiler extension)
+    //u32 depth = START_DEPTH; // an arbitrary starting depth
+    //void * ret_address[MAX_DEPTH]; // used to store "labels as value" (non-standard compiler extension)
 
     // a call site is the start of either an object or an array ('[' or '{')
-    u32 last_loc = 0; // this is the location of the previous call site 
+    //u32 last_loc = 0; // this is the location of the previous call site 
     // (in the tape, at the given depth); 
     // we only need one.
 
@@ -90,23 +90,26 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
     // we can put the current offset into a record for the 
     // scope so we know where it is
 
-    u32 containing_scope_offset[MAX_DEPTH];
+    //u32 containing_scope_offset[MAX_DEPTH];
 
     pj.init();
 
     // add a sentinel to the end to avoid premature exit
     // need to be able to find the \0 at the 'padded length' end of the buffer
     // FIXME: TERRIFYING!
-    size_t j;
-    for (j = len; buf[j] != 0; j++)
-        ;
-    pj.structural_indexes[pj.n_structural_indexes++] = j;
+    //size_t j;
+    //for (j = len; buf[j] != 0; j++)
+    //    ;
+    //pj.structural_indexes[pj.n_structural_indexes++] = j;
 
+// this macro reads the next structural character, updating idx, i and c.
 #define UPDATE_CHAR() { idx = pj.structural_indexes[i++]; c = buf[idx]; DEBUG_PRINTF("Got %c at %d (%d offset)\n", c, idx, i-1);}
 
     // format: call site has 2 entries: 56-bit + '{' or '[' entries pointing first to header then to this location
     //         scope has 2 entries: 56 + '_' entries pointing first to call site then to the last entry in this scope
 
+
+
 #define OPEN_SCOPE() { \
     pj.write_saved_loc(last_loc, pj.save_loc(depth), '_'); \
     pj.write_tape(depth, last_loc, '_'); \
@@ -114,6 +117,7 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
     pj.write_tape(depth, 0, '_'); \
     }
 
+// we are going to increase the depth, we write on the tape the 'c' which is going to be either { or [
 #define ESTABLISH_CALLSITE(RETURN_LABEL, SITE_LABEL) { \
     pj.write_tape(depth, containing_scope_offset[depth], c); \
     last_loc = pj.save_loc(depth); \