CI fixes

.github/workflows/ci.yml

@@ -23,10 +23,7 @@ jobs:
         submodules: recursive
     - run: cmake ${{ matrix.CMAKE_ARGS }} .
     - run: make -j4 VERBOSE=1
-    - if: ${{ ! contains(matrix.CMAKE_ARGS, 'CMAKE_BUILD_TYPE') }}
-      run: make -j4 test ARGS=-V
-    - if: ${{ contains(matrix.CMAKE_ARGS, 'CMAKE_BUILD_TYPE') }}
-      run: ./SMHasher --test=Sanity,Speed,Cyclic,Zeroes,Seed
+    - run: make -j4 test ARGS=-V
   aarch64:
     runs-on: ubuntu-latest
     steps:
@@ -36,9 +33,10 @@ jobs:
         submodules: recursive
     - run: sudo gem install apt-spy2 && sudo apt-spy2 fix
     - run: sudo apt-get update -y
-    - run: sudo apt-get install -o Acquire::Retries=5 -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+    - run: sudo apt-get install -o Acquire::Retries=5 -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu qemu-user-static
     - run: cmake -DCMAKE_TOOLCHAIN_FILE=CMakeARM64Cross.txt
     - run: make -j4 VERBOSE=1
+    - run: make -j4 test ARGS=-V
   armv7:
     runs-on: ubuntu-latest
     steps:
@@ -48,9 +46,10 @@ jobs:
         submodules: recursive
     - run: sudo gem install apt-spy2 && sudo apt-spy2 fix
     - run: sudo apt-get update -y
-    - run: sudo apt-get install -o Acquire::Retries=5 -y gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf
+    - run: sudo apt-get install -o Acquire::Retries=5 -y gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf qemu-user-static
     - run: cmake -DCMAKE_TOOLCHAIN_FILE=CMakeARMCross.txt
     - run: make -j4 VERBOSE=1
+    - run: make -j4 test ARGS=-V
   macOS:
     name: macOS
     runs-on: macOS-latest
@@ -73,7 +72,7 @@ jobs:
         submodules: recursive
     - run: cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_BUILD_TYPE=Release .
     - run: cmake --build . --config Release --verbose
-    - run: Release\SMHasher.exe --test=VerifyAll,Sanity,Speed,Cyclic,Zeroes,Seed
+    - run: ctest -V
   mingw:
     name: mingw
     runs-on: windows-latest

CMakeLists.txt

@@ -60,13 +60,17 @@ elseif ((CMAKE_SYSTEM_PROCESSOR STREQUAL "arm")
   add_definitions(-DHAVE_NEON)
 endif()
 
+if ((CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")   # GNU and others
+    OR (CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")) # macOS
+  set(SYSTEM_PROCESSOR_IS_AARCH64 TRUE)
+else()
+  set(SYSTEM_PROCESSOR_IS_AARCH64 FALSE)
+endif()
+
 # TODO: rather parse `$CC -march=native -dM -E - <<< ''` [GH #10]
 if(NOT CMAKE_CROSSCOMPILING)
   if(CMAKE_SYSTEM_NAME MATCHES "Linux")
-    exec_program(
-      cat ARGS
-      "/proc/cpuinfo"
-      OUTPUT_VARIABLE CPUINFO)
+    file(READ "/proc/cpuinfo" CPUINFO)
     string(REGEX REPLACE "^.*(sse2).*$" "\\1" SSE_THERE "${CPUINFO}")
     string(COMPARE EQUAL "sse2" "${SSE_THERE}" SSE2_TRUE)
     string(REGEX REPLACE "^.*(sse4_2).*$" "\\1" SSE_THERE "${CPUINFO}")
@@ -324,7 +328,7 @@ else(AVX512VL_TRUE)
 endif(AVX512VL_TRUE)
 
 if(CMAKE_CROSSCOMPILING)
-  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+  if(SYSTEM_PROCESSOR_IS_AARCH64)
     set(SSE2_FOUND true)
     set(SSE42_FOUND true)
   elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
@@ -470,13 +474,13 @@ if(SSE42_FOUND)
       list(APPEND X86_64ONLY_SRC metrohash/metrohash64crc.cpp
            metrohash/metrohash128crc.cpp)
     endif()
-    if(NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") AND NOT MSVC)
+    if(NOT SYSTEM_PROCESSOR_IS_AARCH64 AND NOT MSVC)
       list(APPEND X86_64ONLY_SRC crc32_hw1.c)
     endif()
   else()
     message(STATUS "32bit only: CMAKE_SIZEOF_VOID_P=${CMAKE_SIZEOF_VOID_P}")
   endif()
-  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+  if(SYSTEM_PROCESSOR_IS_AARCH64)
     set(SIPHASH_SRC siphash.c)
   else()
     set(SIPHASH_SRC siphash_ssse3.c)
@@ -574,7 +578,7 @@ endif()
 set(BLAKE3_SRC blake3/blake3.c blake3/blake3_dispatch.c
                blake3/blake3_portable.c)
 if(SSE42_FOUND)
-  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+  if(SYSTEM_PROCESSOR_IS_AARCH64)
     set(BLAKE3_SRC ${BLAKE3_SRC} blake3/blake3_neon.c)
   else()
     set(BLAKE3_SRC ${BLAKE3_SRC} blake3/blake3_sse41.c)
@@ -600,6 +604,7 @@ if(SSE42_FOUND
    AND (CMAKE_SIZEOF_VOID_P EQUAL 8)
    AND (NOT MSVC))
   set(UMASH_SRC umash.c umash.hpp)
+  add_definitions(-DHAVE_UMASH)
 endif()
 
 set(ASCON_SRC asconhashv12/hash.c)
@@ -618,7 +623,7 @@ if(CMAKE_CROSSCOMPILING)
   unset(PMPML_SRC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCROSSCOMPILING")
 endif()
-if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+if(SYSTEM_PROCESSOR_IS_AARCH64)
   #HAVE_SSE42
   unset(PMPML_SRC)
 endif()
@@ -752,9 +757,21 @@ SET(exectargets ${exectargets} SMHasher)
 #
 # target_link_libraries( bittest SMHasherSupport ${CMAKE_THREAD_LIBS_INIT} )
 
-if(NOT (CMAKE_CROSSCOMPILING))
+set(QEMU_TESTING FALSE)
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  file(TIMESTAMP "/proc/sys/fs/binfmt_misc/qemu-${CMAKE_SYSTEM_PROCESSOR}" QEMU_TIMESTAMP)
+  if(QEMU_TIMESTAMP) # file exists
+    file(READ "/proc/sys/fs/binfmt_misc/qemu-${CMAKE_SYSTEM_PROCESSOR}" QEMU_BINFMT)
+    if(QEMU_BINFMT MATCHES "^enabled\n")
+      set(QEMU_TESTING TRUE)
+    endif()
+  endif()
+endif()
+
+if(NOT (CMAKE_CROSSCOMPILING) OR QEMU_TESTING)
   enable_testing()
-  add_test(VerifyAll SMHasher --test=VerifyAll)
+  add_test(List SMHasher --list)
+  add_test(VerifyAll SMHasher --test=VerifyAll --verbose)
   add_test(Sanity SMHasher --test=Sanity)
   add_test(Speed SMHasher --test=Speed)
   add_test(Cyclic SMHasher --test=Cyclic)

Hashes.cpp

@@ -388,7 +388,7 @@ FNV128(uint64_t buf[2], const char *key, int len, uint64_t seed)
     uint64_t s0 = multhi;                 // high
     uint64_t s1 = prime128Lower * buf[1]; // low
 
-    s0 += buf[1] << (prime128Shift + prime128Lower * buf[0]);
+    s0 += (buf[1] << prime128Shift) + prime128Lower * buf[0];
 
     // Update the values
     buf[1] = s1;
@@ -818,7 +818,7 @@ void clhash_init()
 {
   void* data = get_random_key_for_clhash(UINT64_C(0xb3816f6a2c68e530), 711);
   memcpy(clhash_random, data, RANDOM_BYTES_NEEDED_FOR_CLHASH);
-  free (data);
+  free_random_key_for_clhash(data);
 }
 bool clhash_bad_seeds(std::vector<uint64_t> &seeds)
 {
@@ -1150,10 +1150,8 @@ void halftime_hash_seed_init(size_t &seed)
 static uint8_t tsip_key[16];
 void tsip_init()
 {
-  uint64_t r = rand_u64();
-  memcpy(&tsip_key[0], &r, 8);
-  r = rand_u64();
-  memcpy(&tsip_key[8], &r, 8);
+  Rand rng(UINT32_C(4044698852));
+  rng.rand_p(tsip_key, sizeof(tsip_key));
 }
 void tsip_test(const void *bytes, int len, uint32_t seed, void *out)
 {
@@ -1278,6 +1276,43 @@ void crc32c_pclmul_test(const void *key, int len, uint32_t seed, void *out)
 #endif
 
 #include "hash-garage/nmhash.h"
+#define NMHASH32_DESC_STR "NMHASH_LITTLE_ENDIAN:" MACRO_ITOA(NMHASH_LITTLE_ENDIAN) ", " \
+                          "NMH_VECTOR:" MACRO_ITOA(NMH_VECTOR) ", " \
+                          "NMH_ACC_ALIGN:" MACRO_ITOA(NMH_ACC_ALIGN) ", " \
+                          "NMH_RESTRICT:" MACRO_ITOA(NMH_RESTRICT)
+const char * const nmhash32_desc("nmhash32, " NMHASH32_DESC_STR);
+const char * const nmhash32x_desc("nmhash32x, " NMHASH32_DESC_STR);
+
+bool nmhash32_broken( void ) {
+  static bool done = false, result;
+  if (done)
+    return result;
+
+  const char entropy[] = "rwgk8M1uxM6XX6c3teQX2yaw8FQWArmcWUSBJ8dcQQJWHYC9Wt2BmpvETxwhYcJTheTbjf49SVRaDJhbEZCq7ki1D6KxpKQSjgwqsiHGSgHLxvPG5kcRnBhjJ1YC8kuh";
+
+  NMH_ALIGN(NMH_ACC_ALIGN) uint32_t accX[sizeof(NMH_ACC_INIT)/sizeof(*NMH_ACC_INIT)];
+  static_assert(sizeof(entropy) >= sizeof(accX), "Need more entropy in entropy[]");
+  memcpy(accX, entropy, sizeof(accX));
+
+  const size_t nbGroups = sizeof(NMH_ACC_INIT) / sizeof(*NMH_ACC_INIT);
+  size_t i;
+
+  for (i = 0; i < nbGroups * 2; ++i) {
+    ((uint16_t*)accX)[i] *= ((uint16_t*)__NMH_M1_V)[i];
+  }
+
+  // XXX: no memory barrier takes place here, just like in NMHASH32_long_round_scalar()
+  // and it affects the `acc` result.
+
+  uint32_t acc = 0;
+  for (i = 0; i < nbGroups; ++i)
+    acc += accX[i];
+
+  result = (acc != UINT32_C(0x249abaee));
+  done = true;
+  return result;
+}
+
 // objsize: 4202f0-420c7d: 2445
 void nmhash32_test ( const void * key, int len, uint32_t seed, void * out ) {
   *(uint32_t*)out = NMHASH32 (key, (const size_t) len, seed);
@@ -1290,6 +1325,14 @@ void nmhash32x_test ( const void * key, int len, uint32_t seed, void * out ) {
 #ifdef HAVE_KHASHV
 #include "k-hashv/khashv.h"
 
+#define KHASH_VER_STR "vector:" MACRO_ITOA(KHASH_VECTOR) ", " \
+                      "scalar:" MACRO_ITOA(KHASHV_SCALAR) ", " \
+                      "__SSE3__:" MACRO_ITOA(__SSE3__) ", " \
+                      "__SSE4_1__:" MACRO_ITOA(__SSE4_1__) ", " \
+                      "__AVX512VL__:" MACRO_ITOA(__AVX512VL__)
+const char * const khashv32_desc("Vectorized K-HashV, 32-bit, " KHASH_VER_STR);
+const char * const khashv64_desc("Vectorized K-HashV, 64-bit, " KHASH_VER_STR);
+
 khashvSeed khashv_seed;
 void khashv_seed_init(size_t &seed) {
   khashv_prep_seed64 (&khashv_seed, seed);
@@ -1304,11 +1347,12 @@ void khashv32_test ( const void *key, int len, uint32_t seed, void *out) {
 }
 #endif // HAVE_KHASHV
 
-PolymurHashParams g_polymurhashparams = {
+#include "polymur-hash/polymur-hash.h"
+static PolymurHashParams g_polymurhashparams = {
   UINT64_C(2172266433527442278), UINT64_C(706663945032637854),
   UINT64_C(754693428422558902),  UINT64_C(9067629717964434866)
 };
-void polymur_seed_init (size_t &seed) {
+void polymur_seed_init (size_t seed) {
   polymur_init_params_from_seed(&g_polymurhashparams,
                                 UINT64_C(0xfedbca9876543210) ^ seed);
 }

Hashes.h

@@ -1156,14 +1156,14 @@ void farsh256_test ( const void * key, int len, unsigned seed, void * out );
 extern "C" {
 #include "blake3/blake3_impl.h"
 // The C API, serially
-  inline void blake3c_test ( const void * key, int len, unsigned seed, void * out )
+  inline void blake3c_test ( const void * key, int len, uint32_t seed, void * out )
   {
     blake3_hasher hasher;
 #if 1
     blake3_hasher_init (&hasher);
     // See GH #168
-    hasher.key[0] ^= (uint32_t)seed;
-    hasher.chunk.cv[0] ^= (uint32_t)seed;
+    hasher.key[0] ^= seed;
+    hasher.chunk.cv[0] ^= seed;
 #else
     // same speed
     uint32_t seed_key[8] = {0x6A09E667 ^ (uint32_t)seed, 0xBB67AE85, 0x3C6EF372,
@@ -1313,6 +1313,9 @@ extern "C" {
   void asconhashv12_256 ( const void * key, int len, uint32_t seed, void * out );
 }
 
+extern const char * const nmhash32_desc;
+extern const char * const nmhash32x_desc;
+bool nmhash32_broken ( void );
 void nmhash32_test ( const void * key, int len, uint32_t seed, void * out );
 void nmhash32x_test ( const void * key, int len, uint32_t seed, void * out );
 
@@ -1389,16 +1392,15 @@ inline void khash64_test ( const void *key, int len, uint32_t seed, void *out) {
 #ifndef HAVE_BIT32
 void khashv_seed_init(size_t &seed);
 // call to khashv_hash_vector not inlined.
+extern const char * const khashv32_desc;
+extern const char * const khashv64_desc;
 void khashv32_test ( const void *key, int len, uint32_t seed, void *out);
 void khashv64_test ( const void *key, int len, uint32_t seed, void *out);
 #endif
 
-#include "polymur-hash/polymur-hash.h"
-extern PolymurHashParams g_polymurhashparams;
-void polymur_seed_init (size_t &seed);
+void polymur_seed_init (size_t seed);
 void polymur_test ( const void *key, int len, uint32_t seed, void *out);
 
-
 /* This version of CRC64 can approach HW crc speeds without hardware support,
    and can be applied to any polynomial. */
 #include "crc64.h"

Types.h

@@ -13,6 +13,11 @@
 using namespace std;
 #include <assert.h>
 
+#define MACRO_ITOA_(x) #x
+#define MACRO_ITOA(x) MACRO_ITOA_(x)
+#define COUNT_OF(x) ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x])))))
+#define ARRAY_END(x) (&(x)[COUNT_OF(x)])
+
 //-----------------------------------------------------------------------------
 // If the optimizer detects that a value in a speed test is constant or unused,
 // the optimizer may remove references to it or otherwise create code that

clhash.c

@@ -12,6 +12,9 @@
 
 #ifdef _WIN32
 #define posix_memalign(p, a, s) (((*(p)) = _aligned_malloc((s), (a))), *(p) ?0 :errno)
+void free_random_key_for_clhash(void *p) { _aligned_free(p); }
+#else
+void free_random_key_for_clhash(void *p) { free(p); }
 #endif
 
 // computes a << 1

clhash.h

@@ -51,6 +51,7 @@ uint64_t clhash(const void* random, const char * stringbyte,
  * Caller is responsible to call "free" on the result.
  */
 void * get_random_key_for_clhash(uint64_t seed1, uint64_t seed2);
+void free_random_key_for_clhash(void *);
 
 #ifdef __cplusplus
 } // extern "C"