From ccc94c9b05a9e89d86ed1123270b60ba1fde20cc Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 29 Jun 2020 21:10:54 -0400 Subject: [PATCH] Mingw tests (32-bit and 64-bit) (#1004) --- .github/workflows/mingw-ci.yml | 54 ++++++++++++++++++++++++++++++ .github/workflows/mingw64-ci.yml | 56 ++++++++++++++++++++++++++++++++ cmake/simdjson-flags.cmake | 3 ++ dependencies/CMakeLists.txt | 1 + singleheader/CMakeLists.txt | 9 +++-- tests/CMakeLists.txt | 5 ++- tests/basictests.cpp | 24 +++++++++++--- tests/jsoncheck.cpp | 2 +- tests/numberparsingcheck.cpp | 10 +++++- tests/parse_many_test.cpp | 2 +- tests/stringparsingcheck.cpp | 2 +- tools/minify.cpp | 2 +- windows/CMakeLists.txt | 2 +- windows/dirent_portable.h | 2 +- windows/getopt.h | 8 +++-- 15 files changed, 165 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/mingw-ci.yml create mode 100644 .github/workflows/mingw64-ci.yml diff --git a/.github/workflows/mingw-ci.yml b/.github/workflows/mingw-ci.yml new file mode 100644 index 00000000..b3048a69 --- /dev/null +++ b/.github/workflows/mingw-ci.yml @@ -0,0 +1,54 @@ +name: MinGW32-CI + +on: push + + +# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both. + +# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So +# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL. + +jobs: + ci: + name: windows-gcc + runs-on: windows-2016 + + env: + CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line + CC: gcc + CXX: g++ + + steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea + - uses: actions/checkout@v2 + + - uses: actions/cache@v2 # we cache the scoop setup with 32-bit GCC + id: cache + with: + path: | + C:\ProgramData\scoop + key: scoop32 # static key: should be good forever + - name: Setup Windows # This should almost never run if the cache works. + if: steps.cache.outputs.cache-hit != 'true' + shell: powershell + run: | + Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh') + scoop install sudo --global + sudo scoop install git --global + sudo scoop install ninja --global + sudo scoop install cmake --global + sudo scoop install gcc --arch 32bit --global + $env:path + Write-Host 'Everything has been installed, you are good!' + - name: Build and Test 32-bit x86 + shell: powershell + run: | + $ENV:PATH="$ENV:PATH;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\shims;C:\Users\runneradmin\scoop\shims" + mkdir build32 + cd build32 + cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake --build . --target basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose + ctest . -R stringparsingcheck --output-on-failure + ctest . -R numberparsingcheck --output-on-failure + ctest . -R errortests --output-on-failure + ctest . -R integer_tests --output-on-failure + ctest . -R pointercheck --output-on-failure diff --git a/.github/workflows/mingw64-ci.yml b/.github/workflows/mingw64-ci.yml new file mode 100644 index 00000000..461e9523 --- /dev/null +++ b/.github/workflows/mingw64-ci.yml @@ -0,0 +1,56 @@ +name: MinGW64-CI + +on: push + + +# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both. + +# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So +# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL. + +jobs: + ci: + name: windows-gcc + runs-on: windows-2016 + + env: + CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line + CC: gcc + CXX: g++ + + steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea + - uses: actions/checkout@v2 + + + - uses: actions/cache@v2 # we cache the scoop setup with 64-bit GCC + id: cache + with: + path: | + C:\ProgramData\scoop + key: scoop64 # static key: should be good forever + - name: Setup Windows # This should almost never run if the cache works. + if: steps.cache.outputs.cache-hit != 'true' + shell: powershell + run: | + Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh') + scoop install sudo --global + sudo scoop install git --global + sudo scoop install ninja --global + sudo scoop install cmake --global + sudo scoop install gcc --arch 64bit --global + $env:path + Write-Host 'Everything has been installed, you are good!' + - name: Build and Test 64-bit x64 + shell: powershell + run: | + $ENV:PATH="$ENV:PATH;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\shims;C:\Users\runneradmin\scoop\shims" + mkdir build64 + cd build64 + cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake --build . --target basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose + ctest . -R stringparsingcheck --output-on-failure + ctest . -R numberparsingcheck --output-on-failure + ctest . -R errortests --output-on-failure + ctest . -R integer_tests --output-on-failure + ctest . -R pointercheck --output-on-failure + diff --git a/cmake/simdjson-flags.cmake b/cmake/simdjson-flags.cmake index cffbac06..3e0f8a46 100644 --- a/cmake/simdjson-flags.cmake +++ b/cmake/simdjson-flags.cmake @@ -53,6 +53,9 @@ endif() option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON) option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" ON) +if(SIMDJSON_COMPETITION) + message(STATUS "Using SIMDJSON_GOOGLE_BENCHMARKS") +endif() set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake") diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index a71797c3..aa2f1ecc 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -18,6 +18,7 @@ if ((Git_FOUND) AND (SIMDJSON_IS_UNDER_GIT)) endfunction(initialize_submodule) if (SIMDJSON_GOOGLE_BENCHMARKS) + message (STATUS "'SIMDJSON_GOOGLE_BENCHMARKS' is requested, configuring..." ) option(BENCHMARK_ENABLE_TESTING OFF) set(BENCHMARK_ENABLE_TESTING OFF) option(BENCHMARK_ENABLE_INSTALL OFF) diff --git a/singleheader/CMakeLists.txt b/singleheader/CMakeLists.txt index f2015939..ca1a6cd5 100644 --- a/singleheader/CMakeLists.txt +++ b/singleheader/CMakeLists.txt @@ -20,7 +20,10 @@ set_source_files_properties(${SINGLEHEADER_FILES} PROPERTIES GENERATED TRUE) find_program(BASH bash) -if (BASH) +# Under Windows, exectuting a bash script works, except that you cannot generally +# do bash C:/path to my script. You need a "mounted" path: /mnt/c/path + +if (BASH AND (NOT WIN32)) add_custom_command( OUTPUT ${SINGLEHEADER_FILES} COMMAND ${CMAKE_COMMAND} -E env @@ -69,7 +72,7 @@ if (BASH) # add_custom_target(amalgamate DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/simdjson.cpp ${CMAKE_CURRENT_SOURCE_DIR}/simdjson.h ${CMAKE_CURRENT_SOURCE_DIR}/amalgamate_demo.cpp ${CMAKE_CURRENT_SOURCE_DIR}/README.md) ## -else(BASH) +else() # We do not have bash, so we use existing amalgamated files instead of generating them ... # (Do not do this if the source and destination are the same!) @@ -83,7 +86,7 @@ else(BASH) ) endif() -endif(BASH) +endif() # # Do not depend on singleheader files directly: depend on this target instead. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5ce27c4e..fa1f89e7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,9 +62,12 @@ add_cpp_test(unicode_tests LABELS acceptance per_implementation) find_program(BASH bash) +# Below we skip anything on Windows, not just visual studio, because running bash under Windows requires you to +# map app paths to their "mounted" equivalent (e.g., /mnt/c/...). So even if you have bash under Windows, extra work would be +# required to make things work robustly. Simply put: bash is not quite portable. # Script tests -if (BASH AND (NOT MSVC) AND (TARGET json2json)) # The scripts are not robust enough to run under Windows even if bash is available +if (BASH AND (NOT WIN32) AND (TARGET json2json)) # The scripts are not robust enough to run under Windows even if bash is available # # json2json test # diff --git a/tests/basictests.cpp b/tests/basictests.cpp index 2d5663d7..279b061c 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -18,6 +18,7 @@ #include "test_macros.h" const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793; +#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, STRINGIFY(x)) namespace number_tests { @@ -68,7 +69,9 @@ namespace number_tests { uint64_t ulp = f64_ulp_dist(actual,expected); if(ulp > maxulp) maxulp = ulp; if(ulp > 0) { - std::cerr << "JSON '" << buf << " parsed to " << actual << " instead of " << expected << std::endl; + std::cerr << "JSON '" << buf << " parsed to "; + fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf + SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD); return false; } } @@ -152,18 +155,25 @@ namespace number_tests { std::cout << __func__ << std::endl; char buf[1024]; simdjson::dom::parser parser; - for (int i = -1000000; i <= 308; ++i) {// large negative values should be zero. + + bool is_pow_correct{1e-308 == std::pow(10,-308)}; + int start_point = is_pow_correct ? -10000 : -307; + if(!is_pow_correct) { + std::cout << "On your system, the pow function is busted. Sorry about that. " << std::endl; + } + for (int i = start_point; i <= 308; ++i) {// large negative values should be zero. size_t n = snprintf(buf, sizeof(buf), "1e%d", i); if (n >= sizeof(buf)) { abort(); } fflush(NULL); - double actual; auto error = parser.parse(buf, n).get(actual); if (error) { std::cerr << error << std::endl; return false; } double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i)); int ulp = (int) f64_ulp_dist(actual, expected); if(ulp > 0) { - std::cerr << "JSON '" << buf << " parsed to " << actual << " instead of " << expected << std::endl; + std::cerr << "JSON '" << buf << " parsed to "; + fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf + SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD); return false; } } @@ -1924,6 +1934,7 @@ namespace format_tests { } } + int main(int argc, char *argv[]) { std::cout << std::unitbuf; int c; @@ -1949,6 +1960,11 @@ int main(int argc, char *argv[]) { if (simdjson::active_implementation->name() == "unsupported") { printf("unsupported CPU\n"); } + // We want to know what we are testing. + std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name(); + std::cout << "(" << simdjson::active_implementation->description() << ")" << std::endl; + std::cout << "------------------------------------------------------------" << std::endl; + std::cout << "Running basic tests." << std::endl; if (validate_tests::run() && minify_tests::run() && diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index 35001c7c..794c5a7f 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -1,5 +1,5 @@ #include -#ifndef _MSC_VER +#if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__)) #include #else // Microsoft can't be bothered to provide standard utils. diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp index dbcc3047..d3d67c87 100644 --- a/tests/numberparsingcheck.cpp +++ b/tests/numberparsingcheck.cpp @@ -9,7 +9,7 @@ #define JSON_TEST_NUMBERS #endif -#ifndef _MSC_VER +#if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__)) #include #else #include @@ -87,7 +87,11 @@ void found_integer(int64_t result, const uint8_t *buf) { char *endptr; long long expected = strtoll((const char *)buf, &endptr, 10); if ((endptr == (const char *)buf) || (expected != result)) { +#if (!(__MINGW32__) && !(__MINGW64__)) fprintf(stderr, "Error: parsed %" PRId64 " out of %.32s, ", result, buf); +#else // mingw is busted since we include #include + fprintf(stderr, "Error: parsed %lld out of %.32s, ", (long long)result, buf); +#endif fprintf(stderr, " while parsing %s \n", fullpath); parse_error |= PARSE_ERROR; } @@ -98,7 +102,11 @@ void found_unsigned_integer(uint64_t result, const uint8_t *buf) { char *endptr; unsigned long long expected = strtoull((const char *)buf, &endptr, 10); if ((endptr == (const char *)buf) || (expected != result)) { +#if (!(__MINGW32__) && !(__MINGW64__)) fprintf(stderr, "Error: parsed %" PRIu64 " out of %.32s, ", result, buf); +#else // mingw is busted since we include #include + fprintf(stderr, "Error: parsed %llu out of %.32s, ", (unsigned long long)result, buf); +#endif fprintf(stderr, " while parsing %s \n", fullpath); parse_error |= PARSE_ERROR; } diff --git a/tests/parse_many_test.cpp b/tests/parse_many_test.cpp index 9e3ab6f4..8add8328 100644 --- a/tests/parse_many_test.cpp +++ b/tests/parse_many_test.cpp @@ -1,5 +1,5 @@ #include -#ifndef _MSC_VER +#if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__)) #include #else // Microsoft can't be bothered to provide standard utils. diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index cefaa637..11976ae1 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -12,7 +12,7 @@ #define JSON_TEST_STRINGS #endif -#ifndef _MSC_VER +#if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__)) #include #else #include diff --git a/tools/minify.cpp b/tools/minify.cpp index 458a9ac2..1a51ff90 100644 --- a/tools/minify.cpp +++ b/tools/minify.cpp @@ -1,5 +1,5 @@ #include -#ifndef _MSC_VER +#if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__)) #include #endif #include diff --git a/windows/CMakeLists.txt b/windows/CMakeLists.txt index 7bbb9122..2be783bc 100644 --- a/windows/CMakeLists.txt +++ b/windows/CMakeLists.txt @@ -1,5 +1,5 @@ add_library(simdjson-windows-headers INTERFACE) -if(MSVC) +if(MSVC OR MINGW) target_include_directories(simdjson-windows-headers INTERFACE .) # getopt.h triggers bogus CRT_SECURE warnings. If you include them, you need this. target_compile_definitions(simdjson-windows-headers INTERFACE _CRT_SECURE_NO_WARNINGS) diff --git a/windows/dirent_portable.h b/windows/dirent_portable.h index 99c28e8c..6d503d81 100644 --- a/windows/dirent_portable.h +++ b/windows/dirent_portable.h @@ -1,7 +1,7 @@ #ifndef SIMDBJSON_DIRENT_PORTABLE_INC_ #define SIMDBJSON_DIRENT_PORTABLE_INC_ -#if (!defined(_WIN32) && !defined(_WIN64)) +#if (!defined(_WIN32) && !defined(_WIN64) && !(__MINGW32__) && !(__MINGW64__)) #include #else #include "toni_ronnko_dirent.h" diff --git a/windows/getopt.h b/windows/getopt.h index 6d1ccc57..33c9e066 100644 --- a/windows/getopt.h +++ b/windows/getopt.h @@ -1,4 +1,5 @@ #ifndef __GETOPT_H__ +/** This file has been modified by D. Lemire for use in simdjson. **/ /* From https://github.com/skandhurkat/Getopt-for-Visual-Studio/blob/master/getopt.h */ /** * DISCLAIMER @@ -56,8 +57,9 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ - +#ifdef _MSC_VER #pragma warning(disable:4996) +#endif #define __GETOPT_H__ @@ -110,7 +112,9 @@ char *optarg; /* argument associated with option */ extern char __declspec(dllimport) *__progname; #endif -#if defined(__CYGWIN__) || defined(__clang__) // D. Lemire (April 2020): adding __clang__ +// D. Lemire (April 2020): adding __clang__ +// D. Lemire (June 2020): adding __MINGW32__ and __MINGW64__ +#if defined(__CYGWIN__) || defined(__clang__) || defined(__MINGW32__) || defined(__MINGW64__) static char EMSG[] = ""; #else #define EMSG ""