jasder
/
simdjson
1
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

We no longer need this old stuff. (#214)

This commit is contained in:
Daniel Lemire 2019-07-09 22:04:02 -04:00 committed by GitHub
parent 4b7e87ec7f
commit 3c1638c046
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
141 changed files with 0 additions and 1265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
scripts/bar.gnuplot
View File

@ -1,27 +0,0 @@
set term pdfcairo fontscale 0.5
set output name
set boxwidth 0.8
set style fill solid
set ylabel "cycles per input byte"
set style line 80 lt rgb "#000000"
# Line style for grid
set style line 81 lt 0 # dashed
set style line 81 lt rgb "#808080" # grey
set grid back linestyle 81
set border 3 back linestyle 80 # Remove border on top and right. These
# borders are useless and make it harder
# to see plotted lines near the border.
# Also, put it in grey; no need for so much emphasis on a border.
set xtics nomirror
set ytics nomirror
set yrange [0:]
set format y "%0.1f";
set style line 1 lt rgb "#A0A0A0" lw 1 pt 1 ps 1
plot filename using 0:2:xtic(1) with boxes notitle ls 1, '' using 0:(1):(sprintf("%.2g", $2)) with labels notitle

12
scripts/data/nuc/allapache_builds.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.128 0.080 1.939 0.127
"RapidJSON " 4.996 0.201 0.441 0.017
"RapidJSON (insitu)" 3.910 0.122 0.563 0.017
"sajson (dynamic mem)" 3.149 0.063 0.699 0.014
"sajson" 2.116 0.058 1.038 0.028
"dropbox (json11) " 14.236 0.137 0.155 0.001
"fastjson " 9.305 0.429 0.237 0.010
"gason " 2.917 0.337 0.754 0.078
"ultrajson " 6.413 0.279 0.344 0.014
"jsmn " 37.170 0.232 0.059 0.000
"cJSON " 8.661 0.091 0.255 0.003

12
scripts/data/nuc/allcanada.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.452 0.050 0.900 0.018
"RapidJSON " 5.084 0.047 0.434 0.004
"RapidJSON (insitu)" 5.067 0.044 0.436 0.004
"sajson (dynamic mem)" 4.375 0.120 0.505 0.013
"sajson" 3.238 0.030 0.682 0.006
"dropbox (json11) " 35.101 0.229 0.063 0.000
"fastjson " 11.541 0.347 0.191 0.006
"gason " 3.489 0.667 0.633 0.102
"ultrajson " 5.596 0.027 0.394 0.002
"jsmn " 382.251 0.900 0.006 0.000
"cJSON " 32.573 0.793 0.068 0.002

12
scripts/data/nuc/allcitm_catalog.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.006 0.033 2.194 0.069
"RapidJSON " 3.265 0.039 0.676 0.008
"RapidJSON (insitu)" 3.100 0.026 0.712 0.006
"sajson (dynamic mem)" 3.344 0.065 0.660 0.013
"sajson" 2.068 0.052 1.067 0.026
"dropbox (json11) " 12.765 0.461 0.173 0.006
"fastjson " 6.849 0.988 0.322 0.041
"gason " 2.292 0.043 0.963 0.018
"ultrajson " 3.819 0.042 0.578 0.006
"jsmn " 12.229 0.078 0.181 0.001
"cJSON " 7.632 0.173 0.289 0.006

12
scripts/data/nuc/allgithub_events.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.016 0.089 2.130 0.174
"RapidJSON " 5.022 0.395 0.438 0.032
"RapidJSON (insitu)" 3.581 0.235 0.613 0.038
"sajson (dynamic mem)" 2.688 0.152 0.815 0.043
"sajson" 2.266 0.095 0.966 0.039
"dropbox (json11) " 16.039 0.246 0.137 0.002
"fastjson " 9.582 0.211 0.230 0.005
"gason " 2.957 0.229 0.741 0.052
"ultrajson " 6.861 0.232 0.321 0.011
"jsmn " 4.499 0.063 0.488 0.007
"cJSON " 8.840 0.163 0.249 0.004

12
scripts/data/nuc/allgsoc-2018.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 0.840 0.030 2.628 0.090
"RapidJSON " 4.617 0.118 0.478 0.012
"RapidJSON (insitu)" 2.835 0.024 0.779 0.006
"sajson (dynamic mem)" 2.356 0.025 0.937 0.010
"sajson" 1.943 0.017 1.136 0.010
"dropbox (json11) " 11.637 0.195 0.190 0.003
"fastjson " 8.641 1.068 0.255 0.028
"gason " 2.588 0.034 0.853 0.011
"ultrajson " 5.439 0.080 0.406 0.006
"jsmn " 14.814 0.094 0.149 0.001
"cJSON " 5.400 0.083 0.409 0.006

12
scripts/data/nuc/allinstruments.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.236 0.083 1.778 0.112
"RapidJSON " 4.238 0.129 0.520 0.015
"RapidJSON (insitu)" 3.932 0.109 0.561 0.015
"sajson (dynamic mem)" 3.807 0.064 0.579 0.010
"sajson" 2.378 0.042 0.926 0.016
"dropbox (json11) " 17.152 0.202 0.129 0.001
"fastjson " 7.785 0.768 0.283 0.025
"gason " 2.822 0.375 0.781 0.092
"ultrajson " 6.106 0.122 0.361 0.007
"jsmn " 9.853 0.081 0.224 0.002
"cJSON " 9.924 0.088 0.222 0.002

12
scripts/data/nuc/allmarine_ik.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.746 0.066 0.804 0.019
"RapidJSON " 6.105 0.050 0.362 0.003
"RapidJSON (insitu)" 5.867 0.039 0.376 0.003
"sajson (dynamic mem)" 5.258 0.333 0.420 0.025
"sajson" 3.555 0.063 0.621 0.011
"dropbox (json11) " 31.563 4.146 0.070 0.008
"fastjson " 12.260 0.409 0.180 0.006
"gason " 3.604 0.943 0.613 0.127
"ultrajson " 7.062 0.066 0.313 0.003
"jsmn " 15.600 0.488 0.142 0.004
"cJSON " 30.865 2.423 0.072 0.005

12
scripts/data/nuc/allmesh.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.742 0.109 0.805 0.031
"RapidJSON " 5.868 0.126 0.376 0.008
"RapidJSON (insitu)" 5.901 0.065 0.374 0.004
"sajson (dynamic mem)" 5.475 0.171 0.403 0.012
"sajson" 3.860 0.087 0.572 0.013
"dropbox (json11) " 28.076 0.352 0.079 0.001
"fastjson " 13.015 0.217 0.170 0.003
"gason " 3.717 0.940 0.594 0.120
"ultrajson " 6.710 0.092 0.329 0.004
"jsmn " 47.987 0.317 0.046 0.000
"cJSON " 35.302 0.433 0.063 0.001

12
scripts/data/nuc/allmesh.pretty.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.816 0.036 1.215 0.023
"RapidJSON " 3.584 0.070 0.616 0.012
"RapidJSON (insitu)" 3.589 0.032 0.615 0.005
"sajson (dynamic mem)" 4.337 0.248 0.509 0.027
"sajson" 2.588 0.036 0.853 0.012
"dropbox (json11) " 16.129 0.201 0.137 0.002
"fastjson " 7.866 0.249 0.281 0.009
"gason " 2.618 0.480 0.843 0.131
"ultrajson " 3.762 0.041 0.587 0.006
"jsmn " 23.841 0.197 0.093 0.001
"cJSON " 17.739 0.260 0.124 0.002

12
scripts/data/nuc/allnumbers.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.292 0.125 0.960 0.050
"RapidJSON " 4.812 0.168 0.458 0.015
"RapidJSON (insitu)" 4.846 0.109 0.455 0.010
"sajson (dynamic mem)" 4.566 0.079 0.482 0.008
"sajson" 3.067 0.063 0.718 0.014
"dropbox (json11) " 25.393 0.190 0.087 0.001
"fastjson " 11.008 0.306 0.200 0.005
"gason " 3.189 0.701 0.690 0.124
"ultrajson " 5.492 0.108 0.401 0.008
"jsmn " 4.822 0.147 0.457 0.013
"cJSON " 29.293 0.500 0.075 0.001

12
scripts/data/nuc/allrandom.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.768 0.068 1.247 0.046
"RapidJSON " 6.532 0.105 0.338 0.005
"RapidJSON (insitu)" 5.397 0.032 0.409 0.002
"sajson (dynamic mem)" 5.900 0.073 0.374 0.005
"sajson" 3.095 0.062 0.713 0.014
"dropbox (json11) " 26.067 1.026 0.085 0.003
"fastjson " 10.810 0.261 0.204 0.005
"gason " 3.417 0.650 0.646 0.103
"ultrajson " 8.801 0.109 0.251 0.003
"jsmn " 79.479 0.333 0.028 0.000
"cJSON " 13.740 0.289 0.161 0.003

12
scripts/data/nuc/alltwitter.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.132 0.066 1.948 0.108
"RapidJSON " 5.186 0.085 0.426 0.007
"RapidJSON (insitu)" 3.759 0.049 0.587 0.008
"sajson (dynamic mem)" 3.639 0.053 0.606 0.009
"sajson" 2.428 0.056 0.909 0.020
"dropbox (json11) " 17.214 0.473 0.128 0.003
"fastjson " 10.042 0.646 0.220 0.013
"gason " 3.227 0.071 0.684 0.015
"ultrajson " 6.217 0.066 0.355 0.004
"jsmn " 9.050 0.091 0.244 0.002
"cJSON " 7.565 0.088 0.292 0.003

12
scripts/data/nuc/alltwitterescaped.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.290 0.070 0.963 0.029
"RapidJSON " 6.236 0.110 0.354 0.006
"RapidJSON (insitu)" 4.400 0.066 0.501 0.007
"sajson (dynamic mem)" 4.176 0.046 0.528 0.006
"sajson" 2.886 0.055 0.764 0.014
"dropbox (json11) " 22.869 0.275 0.097 0.001
"fastjson " 10.673 0.288 0.207 0.005
"gason " 3.841 0.093 0.574 0.014
"ultrajson " 7.277 0.121 0.303 0.005
"jsmn " 9.810 0.100 0.225 0.002
"cJSON " 9.588 0.086 0.230 0.002

12
scripts/data/nuc/allupdate-center.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.430 0.071 1.542 0.073
"RapidJSON " 6.277 0.147 0.352 0.008
"RapidJSON (insitu)" 4.859 0.036 0.454 0.003
"sajson (dynamic mem)" 5.213 0.060 0.423 0.005
"sajson" 2.881 0.063 0.766 0.016
"dropbox (json11) " 22.860 0.369 0.097 0.002
"fastjson " 12.205 0.821 0.181 0.011
"gason " 3.683 0.156 0.599 0.024
"ultrajson " 8.590 0.140 0.257 0.004
"jsmn " 38.342 0.295 0.058 0.000
"cJSON " 10.162 0.127 0.217 0.003

4
scripts/data/nuc/apache_builds.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.126 0.080 1.943 0.128
"RapidJSON (insitu)" 3.873 0.202 0.568 0.028
"sajson" 1.994 0.045 1.102 0.024

36
scripts/data/nuc/bar.gnuplot
View File

@ -1,36 +0,0 @@
set term pdfcairo fontscale 0.65 noenhanced
! python pasteandrotate.py > gbps.txt
set output "gbps.pdf"
set boxwidth 0.8
set style fill solid
set ylabel "parsing speed (GB/s)"
#set style line 80 lt rgb "#000000"
# Line style for grid
set style line 81 lt 0 # dashed
set style line 81 lt rgb "#808080" # grey
set grid back linestyle 81
set border 3 back linestyle 80 # Remove border on top and right. These
# borders are useless and make it harder
# to see plotted lines near the border.
# Also, put it in grey; no need for so much emphasis on a border.
set xtics nomirror
set ytics nomirror
set yrange [0:]
set format y "%0.1f";
set style data histogram
set style histogram cluster gap 1
#set style line 1 lt rgb "#A0A0A0" lw 1 pt 1 ps 1
#set key autotitle columnhead
color1 = "#ff0c18";color2 = "#000000"; color3 = "#0c24ff";
set xtic rotate by 300 scale 1
plot "gbps.txt" using 2:xtic(1) title "simdjson" linecolor rgb color1, \
"" using 3 title "RapidJSON" linecolor rgb color2, \
"" using 4 title "sajson" linecolor rgb color3

4
scripts/data/nuc/canada.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.448 0.049 0.902 0.018
"RapidJSON (insitu)" 5.074 0.038 0.435 0.003
"sajson" 3.264 0.198 0.676 0.039

4
scripts/data/nuc/citm_catalog.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.005 0.030 2.195 0.065
"RapidJSON (insitu)" 3.055 0.087 0.723 0.020
"sajson" 1.999 0.036 1.104 0.019

BIN
scripts/data/nuc/gbps.pdf
View File

Binary file not shown.

4
scripts/data/nuc/github_events.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.031 0.097 2.100 0.177
"RapidJSON (insitu)" 3.551 0.340 0.618 0.054
"sajson" 2.218 0.175 0.987 0.072

4
scripts/data/nuc/gsoc-2018.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 0.842 0.030 2.621 0.090
"RapidJSON (insitu)" 2.840 0.027 0.777 0.007
"sajson" 1.908 0.016 1.157 0.010

4
scripts/data/nuc/instruments.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.236 0.092 1.777 0.122
"RapidJSON (insitu)" 3.853 0.133 0.572 0.019
"sajson" 2.345 0.054 0.939 0.021

4
scripts/data/nuc/marine_ik.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.788 0.069 0.792 0.019
"RapidJSON (insitu)" 5.928 0.062 0.372 0.004
"sajson" 3.602 0.059 0.613 0.010

4
scripts/data/nuc/mesh.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.777 0.122 0.795 0.033
"RapidJSON (insitu)" 5.898 0.116 0.374 0.007
"sajson" 3.774 0.101 0.585 0.015

4
scripts/data/nuc/mesh.pretty.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.817 0.046 1.214 0.030
"RapidJSON (insitu)" 3.576 0.057 0.617 0.010
"sajson" 2.652 0.062 0.832 0.019

4
scripts/data/nuc/numbers.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.453 0.157 0.897 0.054
"RapidJSON (insitu)" 4.856 0.185 0.454 0.017
"sajson" 3.084 0.074 0.714 0.017

28
scripts/data/nuc/pasteandrotate.py
View File

@ -1,28 +0,0 @@
import os
import csv
try: import pandas as pd
except ImportError:
import pip
pip.main(['install', '--user', 'pandas'])
import pandas as pd
def getdata(filename):
df = pd.read_csv(filename, delim_whitespace=True)
return (df["gb_per_s"].tolist())
ourdir=os.path.dirname(os.path.realpath(__file__))
answer = []
for file in os.listdir(ourdir):
if file.startswith("all"):
continue
if file.endswith(".table"):
fullpath = os.path.join(ourdir, file)
answer.append([file[:-11]]+getdata(fullpath))
print("#simdjson RapidJSON sajson")
answer.sort()
for l in answer:
print("\t".join(map(str,l)))

4
scripts/data/nuc/random.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.782 0.084 1.237 0.055
"RapidJSON (insitu)" 5.366 0.090 0.411 0.007
"sajson" 2.978 0.060 0.741 0.015

4
scripts/data/nuc/twitter.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.146 0.065 1.924 0.104
"RapidJSON (insitu)" 3.771 0.061 0.585 0.009
"sajson" 2.392 0.057 0.922 0.021

4
scripts/data/nuc/twitterescaped.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.294 0.069 0.962 0.028
"RapidJSON (insitu)" 4.394 0.064 0.502 0.007
"sajson" 2.827 0.056 0.780 0.015

4
scripts/data/nuc/update-center.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.437 0.078 1.534 0.079
"RapidJSON (insitu)" 4.855 0.104 0.455 0.010
"sajson" 2.772 0.059 0.796 0.017

12
scripts/data/skylake/allapache_builds.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.523 0.034 2.222 0.049
"RapidJSON " 6.546 0.096 0.520 0.008
"RapidJSON (insitu)" 4.917 0.048 0.692 0.007
"sajson (dynamic mem)" 4.116 0.061 0.826 0.012
"sajson" 2.829 0.071 1.200 0.029
"dropbox (json11) " 20.658 0.097 0.165 0.001
"fastjson " 12.749 0.395 0.267 0.008
"gason " 3.639 0.363 0.934 0.084
"ultrajson " 8.329 0.223 0.409 0.011
"jsmn " 50.809 0.150 0.067 0.000
"cJSON " 11.417 0.066 0.298 0.002

12
scripts/data/skylake/allcanada.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.281 0.030 1.038 0.009
"RapidJSON " 6.705 0.017 0.508 0.001
"RapidJSON (insitu)" 6.771 0.011 0.503 0.001
"sajson (dynamic mem)" 5.812 0.100 0.586 0.010
"sajson" 4.250 0.027 0.802 0.005
"dropbox (json11) " 48.060 0.635 0.071 0.001
"fastjson " 16.785 0.175 0.203 0.002
"gason " 3.753 0.748 0.908 0.151
"ultrajson " 7.258 0.019 0.469 0.001
"jsmn " 535.513 0.171 0.006 0.000
"cJSON " 43.412 0.607 0.079 0.001

12
scripts/data/skylake/allcitm_catalog.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.349 0.015 2.524 0.029
"RapidJSON " 4.127 0.016 0.826 0.003
"RapidJSON (insitu)" 3.891 0.013 0.876 0.003
"sajson (dynamic mem)" 3.992 0.017 0.853 0.004
"sajson" 2.968 0.017 1.148 0.007
"dropbox (json11) " 14.293 0.085 0.238 0.001
"fastjson " 9.145 0.977 0.373 0.036
"gason " 2.925 0.069 1.165 0.027
"ultrajson " 4.778 0.024 0.713 0.004
"jsmn " 16.163 0.020 0.211 0.000
"cJSON " 8.935 0.028 0.381 0.001

12
scripts/data/skylake/allgithub_events.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.388 0.059 2.422 0.099
"RapidJSON " 6.727 0.114 0.505 0.008
"RapidJSON (insitu)" 4.529 0.112 0.749 0.018
"sajson (dynamic mem)" 3.526 0.128 0.961 0.034
"sajson" 3.184 0.099 1.064 0.032
"dropbox (json11) " 23.065 0.176 0.148 0.001
"fastjson " 13.145 0.139 0.259 0.003
"gason " 3.634 0.281 0.933 0.067
"ultrajson " 8.912 0.248 0.382 0.010
"jsmn " 5.908 0.076 0.575 0.007
"cJSON " 12.011 0.110 0.283 0.003

12
scripts/data/skylake/allgsoc-2018.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.136 0.022 2.998 0.056
"RapidJSON " 6.166 0.018 0.553 0.002
"RapidJSON (insitu)" 3.409 0.005 1.000 0.002
"sajson (dynamic mem)" 3.174 0.009 1.074 0.003
"sajson" 2.706 0.010 1.259 0.004
"dropbox (json11) " 15.026 0.075 0.227 0.001
"fastjson " 10.953 1.283 0.311 0.033
"gason " 3.121 0.043 1.092 0.015
"ultrajson " 7.697 0.023 0.443 0.001
"jsmn " 20.240 0.040 0.168 0.000
"cJSON " 6.304 0.025 0.541 0.002

12
scripts/data/skylake/allinstruments.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.667 0.041 2.037 0.049
"RapidJSON " 5.548 0.041 0.614 0.004
"RapidJSON (insitu)" 4.909 0.042 0.693 0.006
"sajson (dynamic mem)" 5.102 0.060 0.667 0.008
"sajson" 3.420 0.026 0.995 0.008
"dropbox (json11) " 24.124 0.110 0.141 0.001
"fastjson " 10.926 0.143 0.312 0.004
"gason " 3.445 0.517 0.987 0.129
"ultrajson " 7.649 0.109 0.445 0.006
"jsmn " 13.093 0.050 0.260 0.001
"cJSON " 13.135 0.067 0.259 0.001

12
scripts/data/skylake/allmarine_ik.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.702 0.046 0.921 0.011
"RapidJSON " 7.619 0.036 0.447 0.002
"RapidJSON (insitu)" 7.426 0.014 0.459 0.001
"sajson (dynamic mem)" 6.702 0.145 0.508 0.011
"sajson" 5.006 0.054 0.681 0.007
"dropbox (json11) " 45.202 4.961 0.075 0.007
"fastjson " 17.095 0.363 0.199 0.004
"gason " 4.498 1.051 0.758 0.143
"ultrajson " 8.591 0.032 0.397 0.001
"jsmn " 18.672 0.072 0.183 0.001
"cJSON " 43.523 1.752 0.078 0.003

12
scripts/data/skylake/allmesh.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.715 0.053 0.917 0.013
"RapidJSON " 7.616 0.069 0.447 0.004
"RapidJSON (insitu)" 7.686 0.035 0.443 0.002
"sajson (dynamic mem)" 6.644 0.142 0.513 0.011
"sajson" 4.930 0.069 0.691 0.010
"dropbox (json11) " 39.737 0.247 0.086 0.001
"fastjson " 18.445 0.106 0.185 0.001
"gason " 4.709 1.019 0.723 0.129
"ultrajson " 8.493 0.050 0.401 0.002
"jsmn " 66.500 0.097 0.051 0.000
"cJSON " 48.582 0.207 0.070 0.000

12
scripts/data/skylake/allmesh.pretty.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.430 0.023 1.402 0.013
"RapidJSON " 4.683 0.032 0.728 0.005
"RapidJSON (insitu)" 4.703 0.011 0.724 0.002
"sajson (dynamic mem)" 5.078 0.312 0.671 0.039
"sajson" 3.410 0.024 0.999 0.007
"dropbox (json11) " 21.818 0.103 0.156 0.001
"fastjson " 11.126 0.167 0.306 0.005
"gason " 3.329 0.506 1.024 0.135
"ultrajson " 4.810 0.026 0.708 0.004
"jsmn " 32.929 0.169 0.103 0.001
"cJSON " 23.476 0.105 0.145 0.001

12
scripts/data/skylake/allnumbers.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.163 0.036 1.075 0.012
"RapidJSON " 6.369 0.090 0.534 0.007
"RapidJSON (insitu)" 6.471 0.035 0.526 0.003
"sajson (dynamic mem)" 6.007 0.058 0.566 0.005
"sajson" 4.074 0.075 0.834 0.015
"dropbox (json11) " 37.788 0.298 0.090 0.001
"fastjson " 15.389 0.151 0.221 0.002
"gason " 4.105 0.707 0.828 0.121
"ultrajson " 7.156 0.070 0.476 0.005
"jsmn " 6.077 0.036 0.560 0.003
"cJSON " 42.276 0.153 0.081 0.000

23
scripts/data/skylake/allpasteandrotate.py
View File

@ -1,23 +0,0 @@
import os
import csv
try: import pandas as pd
except ImportError:
import pip
pip.main(['install', '--user', 'pandas'])
import pandas as pd
def getdata(filename):
df = pd.read_csv(filename, delim_whitespace=True)
return (df["gb_per_s"].tolist())
ourdir=os.path.dirname(os.path.realpath(__file__))
answer = []
for file in os.listdir(ourdir):
if file.startswith("all") and file.endswith(".table"):
fullpath = os.path.join(ourdir, file)
answer.append([file[3:-11]]+getdata(fullpath))
print(" \t&\t simdjson \t&\t RapidJSON \t&\t RapidJSONinsitu \t&\t sajsondyn \t&\t sajson \t&\t dropbox-json11 \t&\t fastjson \t&\t gason \t&\t ultrajson \t&\t jsmn\t&\t cJSON \\\\")
answer.sort()
for l in answer:
print("\t&\t".join(map(lambda x : (('{:.2f}'.format(x) if x < 1 else '{:.1f}'.format(x) ) if (type(x) is float) else x),l))+"\\\\")

12
scripts/data/skylake/allrandom.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.394 0.041 1.422 0.024
"RapidJSON " 8.428 0.044 0.404 0.002
"RapidJSON (insitu)" 6.850 0.023 0.497 0.002
"sajson (dynamic mem)" 7.760 0.044 0.439 0.002
"sajson" 4.153 0.039 0.820 0.008
"dropbox (json11) " 33.377 0.126 0.102 0.000
"fastjson " 15.233 0.209 0.224 0.003
"gason " 4.156 0.719 0.819 0.121
"ultrajson " 11.560 0.082 0.295 0.002
"jsmn " 111.680 1.493 0.031 0.000
"cJSON " 17.926 0.175 0.190 0.002

12
scripts/data/skylake/alltwitter.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.560 0.030 2.182 0.042
"RapidJSON " 6.671 0.041 0.511 0.003
"RapidJSON (insitu)" 4.772 0.027 0.714 0.004
"sajson (dynamic mem)" 4.883 0.046 0.698 0.006
"sajson" 3.502 0.027 0.973 0.008
"dropbox (json11) " 24.198 0.143 0.141 0.001
"fastjson " 13.158 0.907 0.259 0.017
"gason " 4.026 0.097 0.846 0.020
"ultrajson " 8.097 0.047 0.421 0.002
"jsmn " 12.138 0.064 0.281 0.001
"cJSON " 10.033 0.054 0.340 0.002

12
scripts/data/skylake/alltwitterescaped.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.116 0.037 1.093 0.013
"RapidJSON " 8.138 0.037 0.419 0.002
"RapidJSON (insitu)" 5.738 0.027 0.594 0.003
"sajson (dynamic mem)" 5.670 0.041 0.601 0.004
"sajson" 3.945 0.040 0.863 0.009
"dropbox (json11) " 31.865 0.152 0.107 0.001
"fastjson " 14.110 0.939 0.241 0.015
"gason " 4.759 0.091 0.716 0.013
"ultrajson " 9.552 0.062 0.357 0.002
"jsmn " 13.007 0.167 0.262 0.003
"cJSON " 12.678 0.057 0.269 0.001

12
scripts/data/skylake/allupdate-center.json.table
View File

@ -1,12 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.959 0.036 1.738 0.031
"RapidJSON " 8.469 0.050 0.402 0.002
"RapidJSON (insitu)" 6.231 0.026 0.547 0.002
"sajson (dynamic mem)" 7.073 0.038 0.482 0.003
"sajson" 4.036 0.039 0.844 0.008
"dropbox (json11) " 31.284 0.113 0.109 0.000
"fastjson " 16.820 0.969 0.203 0.011
"gason " 4.733 0.211 0.719 0.031
"ultrajson " 11.067 0.082 0.308 0.002
"jsmn " 53.609 0.470 0.064 0.001
"cJSON " 13.556 0.095 0.251 0.002

4
scripts/data/skylake/apache_builds.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.543 0.033 2.194 0.046
"RapidJSON (insitu)" 4.902 0.046 0.694 0.007
"sajson" 2.753 0.151 1.233 0.064

36
scripts/data/skylake/bar.gnuplot
View File

@ -1,36 +0,0 @@
set term pdfcairo fontscale 0.65 noenhanced
! python pasteandrotate.py > gbps.txt
set output "gbps.pdf"
set boxwidth 0.8
set style fill solid
set ylabel "parsing speed (GB/s)"
#set style line 80 lt rgb "#000000"
# Line style for grid
set style line 81 lt 0 # dashed
set style line 81 lt rgb "#808080" # grey
set grid back linestyle 81
set border 3 back linestyle 80 # Remove border on top and right. These
# borders are useless and make it harder
# to see plotted lines near the border.
# Also, put it in grey; no need for so much emphasis on a border.
set xtics nomirror
set ytics nomirror
set yrange [0:]
set format y "%0.1f";
set style data histogram
set style histogram cluster gap 1
#set style line 1 lt rgb "#A0A0A0" lw 1 pt 1 ps 1
#set key autotitle columnhead
color1 = "#ff0c18";color2 = "#000000"; color3 = "#0c24ff";
set xtic rotate by 300 scale 1
plot "gbps.txt" using 2:xtic(1) title "simdjson" linecolor rgb color1, \
"" using 3 title "RapidJSON" linecolor rgb color2, \
"" using 4 title "sajson" linecolor rgb color3

4
scripts/data/skylake/canada.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.289 0.040 1.036 0.012
"RapidJSON (insitu)" 6.741 0.026 0.506 0.002
"sajson" 4.562 0.024 0.747 0.004

4
scripts/data/skylake/citm_catalog.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.385 0.017 2.458 0.029
"RapidJSON (insitu)" 3.700 0.012 0.921 0.003
"sajson" 2.916 0.024 1.168 0.010

BIN
scripts/data/skylake/gbps.pdf
View File

Binary file not shown.

4
scripts/data/skylake/github_events.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.381 0.048 2.435 0.080
"RapidJSON (insitu)" 4.592 0.130 0.739 0.020
"sajson" 3.220 0.113 1.052 0.035

4
scripts/data/skylake/gsoc-2018.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.128 0.021 3.020 0.056
"RapidJSON (insitu)" 3.467 0.007 0.983 0.002
"sajson" 2.699 0.010 1.262 0.005

4
scripts/data/skylake/instruments.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.671 0.039 2.032 0.046
"RapidJSON (insitu)" 4.911 0.056 0.693 0.008
"sajson" 3.524 0.033 0.965 0.009

4
scripts/data/skylake/marine_ik.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.691 0.054 0.923 0.013
"RapidJSON (insitu)" 7.533 0.035 0.452 0.002
"sajson" 5.006 0.223 0.681 0.029

4
scripts/data/skylake/mesh.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.703 0.056 0.920 0.014
"RapidJSON (insitu)" 7.820 0.069 0.436 0.004
"sajson" 5.245 0.059 0.650 0.007

4
scripts/data/skylake/mesh.pretty.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.430 0.022 1.402 0.013
"RapidJSON (insitu)" 4.764 0.031 0.715 0.005
"sajson" 3.522 0.021 0.968 0.006

4
scripts/data/skylake/numbers.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.123 0.036 1.088 0.012
"RapidJSON (insitu)" 6.568 0.094 0.518 0.007
"sajson" 4.182 0.034 0.813 0.007

28
scripts/data/skylake/pasteandrotate.py
View File

@ -1,28 +0,0 @@
import os
import csv
try: import pandas as pd
except ImportError:
import pip
pip.main(['install', '--user', 'pandas'])
import pandas as pd
def getdata(filename):
df = pd.read_csv(filename, delim_whitespace=True)
return (df["gb_per_s"].tolist())
ourdir=os.path.dirname(os.path.realpath(__file__))
answer = []
for file in os.listdir(ourdir):
if file.startswith("all"):
continue
if file.endswith(".table"):
fullpath = os.path.join(ourdir, file)
answer.append([file[:-11]]+getdata(fullpath))
print("#simdjson RapidJSON sajson")
answer.sort()
for l in answer:
print("\t".join(map(str,l)))

4
scripts/data/skylake/random.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 2.401 0.040 1.418 0.023
"RapidJSON (insitu)" 6.899 0.030 0.494 0.002
"sajson" 4.167 0.043 0.817 0.008

4
scripts/data/skylake/twitter.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.554 0.032 2.190 0.044
"RapidJSON (insitu)" 4.859 0.028 0.701 0.004
"sajson" 3.451 0.083 0.987 0.023

4
scripts/data/skylake/twitterescaped.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 3.116 0.036 1.093 0.013
"RapidJSON (insitu)" 5.613 0.041 0.607 0.004
"sajson" 4.005 0.033 0.850 0.007

4
scripts/data/skylake/update-center.json.table
View File

@ -1,4 +0,0 @@
name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err
"simdjson " 1.956 0.036 1.740 0.032
"RapidJSON (insitu)" 6.049 0.039 0.563 0.004
"sajson" 3.961 0.050 0.860 0.011

20
scripts/forpaper.sh
View File

@ -1,20 +0,0 @@
#!/bin/bash
[[ "$(command -v gnuplot)" ]] || { echo "gnuplot is not installed" 1>&2 ; exit 1; }
python -c "import pandas"
if [ $? -ne 0 ]; then
echo "pandas is not installed, try pip install pandas" 1>&2 ; exit 1;
fi
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
cd $SCRIPTPATH/..; make clean
cd $SCRIPTPATH/setupfortesting/; sudo ./setupfortesting.sh
$SCRIPTPATH/parsingcompdata.sh
$SCRIPTPATH/plotparse.sh
$SCRIPTPATH/statisticalmodel.sh

31
scripts/modeldata/nuc/analysis.txt
View File

@ -1,31 +0,0 @@
loading modeltable.txt
integer_count float_count ... total_cycles ratio
count 14.000000 14.000000 ... 1.400000e+01 14.000000
mean 17156.642857 21488.071429 ... 2.910166e+06 2.517336
std 35520.191644 40466.023650 ... 3.465531e+06 1.043643
min 0.000000 0.000000 ... 9.816417e+04 1.209798
25% 13.000000 0.000000 ... 6.484984e+05 1.643587
50% 2108.000000 0.500000 ... 1.607978e+06 2.305758
75% 12044.500000 26800.250000 ... 3.766565e+06 3.436660
max 130225.000000 114950.000000 ... 1.227446e+07 4.129712
[8 rows x 18 columns]
chosenpredictors= ['integer_count', 'float_count', 'string_count', 'backslash_count', 'nonasciibyte_count', 'object_count', 'array_count', 'null_count', 'true_count', 'false_count', 'byte_count', 'structural_indexes_count']
target = stage1_cycle_count
1.8 cycles per structural_indexes_count
0.62 cycles per byte_count
R2 = 0.9966890133532899
target = stage2_cycle_count
19 cycles per float_count
9.5 cycles per structural_indexes_count
0.33 cycles per byte_count
R2 = 0.9868882924152415
target = total_cycles
19 cycles per float_count
11 cycles per structural_indexes_count
0.95 cycles per byte_count
R2 = 0.9923672903089373

64
scripts/modeldata/nuc/learn.py
View File

@ -1,64 +0,0 @@
import os
import pandas as pd
import numpy as np
#import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.preprocessing import normalize
from sklearn import metrics
def displaycoefs(coef_name):
coef_name.sort()
coef_name.reverse()
for c,n in coef_name:
print("\t%0.2g cycles per %s "%(c,n))
datafile = "modeltable.txt" ## from ./scripts/statisticalmodel.sh
predictors = ["integer_count", "float_count", "string_count", "backslash_count", "nonasciibyte_count", "object_count", "array_count", "null_count", "true_count", "false_count", "byte_count", "structural_indexes_count"]
targets = ["stage1_cycle_count", "stage1_instruction_count", "stage2_cycle_count", "stage2_instruction_count"]
print("loading", datafile)
dataset = pd.read_csv(datafile, delim_whitespace=True, skip_blank_lines=True, comment="#", header=None, names = predictors + targets)
dataset.columns = predictors + targets
dataset['total_cycles']=dataset['stage1_cycle_count']+dataset['stage2_cycle_count']
dataset['ratio']=dataset['total_cycles']/dataset['byte_count']
#print(dataset[['ratio']])
print(dataset.describe())
chosenpredictors = predictors
print("chosenpredictors=",chosenpredictors)
print()
chosentargets=["stage1_cycle_count", "stage2_cycle_count","total_cycles"]
for t in chosentargets:
print("target = ", t)
howmany = 1 # we want at most one predictors
if(t.startswith("stage1")):
howmany = 2 # we allow for less
if(t.startswith("stage2")):
howmany = 3 # we allow for more
if(t.startswith("total")):
howmany = 3 # we allow for more
A=10000000.0
while(True):
regressor = Lasso(max_iter=100000, alpha=A, positive = True, normalize=False, fit_intercept=False) #LinearRegression(normalize=False, fit_intercept=False)
x = dataset[chosenpredictors]
y = dataset[[t]]
regressor.fit(x, y)
rest = list(filter(lambda z: z[0] != 0, zip(regressor.coef_,chosenpredictors) ))
nonzero = len(rest)
if(nonzero > howmany):
A *= 1.2
else:
#print(rest)
displaycoefs(rest)
print("R2 = ", regressor.score(x,y))
Y_pred = regressor.predict(x)
break
print()

43
scripts/modeldata/nuc/modeltable.txt
View File

@ -1,43 +0,0 @@
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/apache_builds.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
2 0 5289 26 0 884 3 0 2 1 127275 12365 96850.637000 348196.030000 112942.436000 373481.072000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/canada.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
46 111080 12 0 0 4 56045 0 0 0 2251051 334374 2087660.260000 6948305.160000 5956295.900000 21752650.940000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/citm_catalog.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
14392 0 26604 2 348 10937 10451 1263 0 0 1727204 135991 1238270.780000 4449267.000000 1280918.100000 4608414.520000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/github_events.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
149 0 1891 155 4 180 19 24 57 7 65132 4657 46894.246000 165589.015000 51428.255000 156822.031000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/gsoc-2018.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
0 0 34128 15023 0 3793 0 0 0 0 3327831 75842 2176206.820000 6652268.140000 1892621.160000 4227820.800000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/instruments.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
4935 0 6889 0 0 1012 194 431 17 109 220346 27174 172724.501000 619214.084000 221539.929000 793915.124000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/marine_ik.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
130225 114950 38268 0 0 9680 28377 0 6 0 2983466 643013 3017329.300000 10443612.460000 8894712.340000 29274112.300000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/mesh.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
40613 32400 11 0 0 3 3610 0 0 0 723597 153275 713256.527000 2392843.368000 2207801.598000 7862782.005000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/mesh.pretty.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
40613 32400 11 0 0 3 3610 0 0 0 1577353 153275 1175555.160000 4272568.300000 3032006.520000 9753016.920000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/numbers.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
0 10001 0 0 0 0 1 0 0 0 150124 20004 118329.401000 425024.036000 379803.665000 1291145.189000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/random.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
5002 0 33005 0 103482 4001 1001 0 495 505 510476 88018 577931.349000 2045179.271000 729482.007000 2515740.428000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/twitterescaped.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
2108 1 18099 33048 0 1264 1050 1946 345 2446 562408 55264 445053.778000 1469947.162000 1423606.912000 4098363.635000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/twitter.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
2108 1 18099 1230 95406 1264 1050 1946 345 2446 631514 55264 528976.831000 1821904.235000 508297.903000 1604709.304000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/update-center.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
0 0 27229 284 49 1896 1937 0 134 252 533178 63420 459986.211000 1489108.202000 638098.347000 1873065.386000

1
scripts/modeldata/nuc/runanalysis.sh
View File

@ -1 +0,0 @@
python learn.py > analysis.txt

31
scripts/modeldata/skylake/analysis.txt
View File

@ -1,31 +0,0 @@
loading modeltable.txt
integer_count float_count ... total_cycles ratio
count 14.000000 14.000000 ... 1.400000e+01 14.000000
mean 17156.642857 21488.071429 ... 2.898374e+06 2.523610
std 35520.191644 40466.023650 ... 3.408262e+06 1.021949
min 0.000000 0.000000 ... 9.934419e+04 1.225950
25% 13.000000 0.000000 ... 6.545444e+05 1.650991
50% 2108.000000 0.500000 ... 1.611746e+06 2.369115
75% 12044.500000 26800.250000 ... 3.803468e+06 3.441740
max 130225.000000 114950.000000 ... 1.205456e+07 4.110868
[8 rows x 18 columns]
chosenpredictors= ['integer_count', 'float_count', 'string_count', 'backslash_count', 'nonasciibyte_count', 'object_count', 'array_count', 'null_count', 'true_count', 'false_count', 'byte_count', 'structural_indexes_count']
target = stage1_cycle_count
1.9 cycles per structural_indexes_count
0.63 cycles per byte_count
R2 = 0.9965695015271681
target = stage2_cycle_count
19 cycles per float_count
9 cycles per structural_indexes_count
0.36 cycles per byte_count
R2 = 0.9858116267470738
target = total_cycles
19 cycles per float_count
11 cycles per structural_indexes_count
0.98 cycles per byte_count
R2 = 0.9919590553913162

64
scripts/modeldata/skylake/learn.py
View File

@ -1,64 +0,0 @@
import os
import pandas as pd
import numpy as np
#import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.preprocessing import normalize
from sklearn import metrics
def displaycoefs(coef_name):
coef_name.sort()
coef_name.reverse()
for c,n in coef_name:
print("\t%0.2g cycles per %s "%(c,n))
datafile = "modeltable.txt" ## from ./scripts/statisticalmodel.sh
predictors = ["integer_count", "float_count", "string_count", "backslash_count", "nonasciibyte_count", "object_count", "array_count", "null_count", "true_count", "false_count", "byte_count", "structural_indexes_count"]
targets = ["stage1_cycle_count", "stage1_instruction_count", "stage2_cycle_count", "stage2_instruction_count"]
print("loading", datafile)
dataset = pd.read_csv(datafile, delim_whitespace=True, skip_blank_lines=True, comment="#", header=None, names = predictors + targets)
dataset.columns = predictors + targets
dataset['total_cycles']=dataset['stage1_cycle_count']+dataset['stage2_cycle_count']
dataset['ratio']=dataset['total_cycles']/dataset['byte_count']
#print(dataset[['ratio']])
print(dataset.describe())
chosenpredictors = predictors
print("chosenpredictors=",chosenpredictors)
print()
chosentargets=["stage1_cycle_count", "stage2_cycle_count","total_cycles"]
for t in chosentargets:
print("target = ", t)
howmany = 1 # we want at most one predictors
if(t.startswith("stage1")):
howmany = 2 # we allow for less
if(t.startswith("stage2")):
howmany = 3 # we allow for more
if(t.startswith("total")):
howmany = 3 # we allow for more
A=10000000.0
while(True):
regressor = Lasso(max_iter=100000, alpha=A, positive = True, normalize=False, fit_intercept=False) #LinearRegression(normalize=False, fit_intercept=False)
x = dataset[chosenpredictors]
y = dataset[[t]]
regressor.fit(x, y)
rest = list(filter(lambda z: z[0] != 0, zip(regressor.coef_,chosenpredictors) ))
nonzero = len(rest)
if(nonzero > howmany):
A *= 1.2
else:
#print(rest)
displaycoefs(rest)
print("R2 = ", regressor.score(x,y))
Y_pred = regressor.predict(x)
break
print()

43
scripts/modeldata/skylake/modeltable.txt
View File

@ -1,43 +0,0 @@
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/apache_builds.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
2 0 5289 26 0 884 3 0 2 1 127275 12365 98860.705000 352170.006000 110092.182000 368181.042000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/canada.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
46 111080 12 0 0 4 56045 0 0 0 2251051 334374 2117954.980000 7018646.660000 5982086.300000 21368228.540000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/citm_catalog.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
14392 0 26604 2 348 10937 10451 1263 0 0 1727204 135991 1256015.620000 4503327.740000 1260240.200000 4555510.160000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/github_events.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
149 0 1891 155 4 180 19 24 57 7 65132 4657 48294.810000 167625.000000 51049.382000 154544.023000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/gsoc-2018.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
0 0 34128 15023 0 3793 0 0 0 0 3327831 75842 2222068.100000 6756257.580000 1857685.960000 4189900.300000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/instruments.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
4935 0 6889 0 0 1012 194 431 17 109 220346 27174 179195.077000 626094.037000 219242.424000 781025.075000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/marine_ik.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
130225 114950 38268 0 0 9680 28377 0 6 0 2983466 643013 3082930.040000 10536841.740000 8971625.320000 28934128.020000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/mesh.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
40613 32400 11 0 0 3 3610 0 0 0 723597 153275 692445.484000 2415449.195000 2282166.283000 7726735.452000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/mesh.pretty.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
40613 32400 11 0 0 3 3610 0 0 0 1577353 153275 1199740.400000 4321852.120000 2992529.340000 9632717.140000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/numbers.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
0 10001 0 0 0 0 1 0 0 0 150124 20004 125122.630000 429712.029000 394222.012000 1261093.069000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/random.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
5002 0 33005 0 103482 4001 1001 0 495 505 510476 88018 594131.429000 2066269.128000 723559.441000 2475732.242000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/twitterescaped.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
2108 1 18099 33048 0 1264 1050 1946 345 2446 562408 55264 482727.010000 1487521.087000 1423074.231000 4071016.251000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/twitter.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
2108 1 18099 1230 95406 1264 1050 1946 345 2446 631514 55264 550609.995000 1843920.091000 509533.657000 1577362.190000
#/home/dlemire/CVS/github/simdjson/scripts/../jsonexamples/update-center.json
# integer_count float_count string_count backslash_count nonasciibyte_count object_count array_count null_count true_count false_count byte_count structural_indexes_count stage1_cycle_count stage1_instruction_count stage2_cycle_count stage2_instruction_count stage3_cycle_count stage3_instruction_count
0 0 27229 284 49 1896 1937 0 134 252 533178 63420 495267.828000 1505783.095000 654760.566000 1842005.228000

1
scripts/modeldata/skylake/runanalysis.sh
View File

@ -1 +0,0 @@
python learn.py > analysis.txt

BIN
scripts/plots/nuc/apache_builds.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/apache_builds.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/canada.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/canada.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/citm_catalog.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/citm_catalog.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/github_events.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/github_events.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/gsoc-2018.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/gsoc-2018.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/instruments.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/instruments.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/marine_ik.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/marine_ik.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/mesh.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/mesh.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/mesh.pretty.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/mesh.pretty.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/numbers.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/numbers.jsonparseandstat.pdf
View File

Binary file not shown.

15
scripts/plots/nuc/parselinuxtable.txt
View File

@ -1,15 +0,0 @@
"apache_builds" 0.005952 0.763112 0.878737 1.647801 "apache_builds" 0.006006 0.752234 0.879753 1.637993 "apache_builds" 0.005884 0.759751 0.879309 1.644945 "apache_builds" 0.006444 0.763598 0.350529 1.120571
"canada" 0.001381 0.957843 2.662114 3.621338 "canada" 0.001467 0.927421 2.675200 3.604088 "canada" 0.001315 0.937992 0.821737 1.761044 "canada" 0.001417 0.951059 2.667583 3.620060
"citm_catalog" 0.001336 0.738699 0.780890 1.520924 "citm_catalog" 0.001531 0.732052 0.774012 1.507595 "citm_catalog" 0.001545 0.736709 0.567611 1.305866 "citm_catalog" 0.001795 0.735362 0.595379 1.332536
"github_events" 0.010639 0.721886 0.769225 1.501750 "github_events" 0.011586 0.709699 0.784076 1.505361 "github_events" 0.010992 0.720799 0.729923 1.461713 "github_events" 0.011511 0.724542 0.332537 1.068591
"gsoc-2018" 0.000787 0.669334 0.627500 1.297620 "gsoc-2018" 0.000788 0.623104 0.614476 1.238368 "gsoc-2018" 0.000779 0.672922 0.614661 1.288362 "gsoc-2018" 0.000714 0.663066 0.155355 0.819135
"instruments" 0.003500 0.782957 1.012730 1.799187 "instruments" 0.003462 0.777753 1.010707 1.791923 "instruments" 0.003464 0.778729 0.792333 1.574526 "instruments" 0.003592 0.784070 0.634848 1.422510
"marine_ik" 0.000901 1.047364 3.045017 4.093284 "marine_ik" 0.001029 1.027565 3.047477 4.076071 "marine_ik" 0.000907 1.038903 1.066186 2.105996 "marine_ik" 0.001012 1.044582 2.940801 3.986395
"mesh" 0.001103 0.982993 3.075482 4.059578 "mesh" 0.001247 0.914444 3.047722 3.963413 "mesh" 0.001086 0.900909 0.798541 1.700535 "mesh" 0.001333 0.973654 3.056232 4.031219
"mesh.pretty" 0.001582 0.763502 1.903150 2.668234 "mesh.pretty" 0.001496 0.763421 1.910400 2.675316 "mesh.pretty" 0.001670 0.769271 0.451983 1.222924 "mesh.pretty" 0.001422 0.769347 1.901950 2.672719
"numbers" 0.005106 0.786358 2.533630 3.325094 "numbers" 0.005326 0.800752 2.526423 3.332501 "numbers" 0.005017 0.786403 0.473920 1.265340 "numbers" 0.005265 0.787399 2.545165 3.337829
"random" 0.001493 1.133635 1.424102 2.559230 "random" 0.001585 0.900653 1.429734 2.331971 "random" 0.001548 1.147079 1.356399 2.505026 "random" 0.001564 1.133086 0.707339 1.841989
"twitterescaped" 0.001441 0.788297 2.528283 3.318021 "twitterescaped" 0.001504 0.768639 2.525659 3.295801 "twitterescaped" 0.001413 0.787999 2.446287 3.235699 "twitterescaped" 0.001527 0.782157 0.436354 1.220039
"twitter" 0.001342 0.839178 0.801661 1.642181 "twitter" 0.001323 0.751497 0.793082 1.545902 "twitter" 0.001233 0.853322 0.732146 1.586701 "twitter" 0.001294 0.834811 0.391635 1.227740
"update-center" 0.001663 0.864404 1.194040 2.060107 "update-center" 0.001533 0.853176 1.195099 2.049808 "update-center" 0.001507 0.864929 1.194072 2.060508 "update-center" 0.001571 0.855218 0.540045 1.396834

BIN
scripts/plots/nuc/random.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/random.jsonparseandstat.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/stackedperf.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/twitter.json.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/twitter.jsondistinctuserid.pdf
View File

Binary file not shown.

BIN
scripts/plots/nuc/twitter.jsonparseandstat.pdf
View File

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More