40 # if defined(_M_X64) && !defined(_M_ARM64EC)
41 # pragma intrinsic(_umul128)
50 # define RAPIDHASH_ALWAYS_INLINE __forceinline
51 #elif defined(__GNUC__)
52 # define RAPIDHASH_ALWAYS_INLINE inline __attribute__((__always_inline__))
54 # define RAPIDHASH_ALWAYS_INLINE inline
58 # define RAPIDHASH_NOEXCEPT noexcept
59 # define RAPIDHASH_CONSTEXPR constexpr
60 # ifndef RAPIDHASH_INLINE
61 # define RAPIDHASH_INLINE RAPIDHASH_ALWAYS_INLINE
63 # if __cplusplus >= 201402L && !defined(_MSC_VER)
64 # define RAPIDHASH_INLINE_CONSTEXPR RAPIDHASH_ALWAYS_INLINE constexpr
66 # define RAPIDHASH_INLINE_CONSTEXPR RAPIDHASH_ALWAYS_INLINE
69 # define RAPIDHASH_NOEXCEPT
70 # define RAPIDHASH_CONSTEXPR static const
71 # ifndef RAPIDHASH_INLINE
72 # define RAPIDHASH_INLINE static RAPIDHASH_ALWAYS_INLINE
74 # define RAPIDHASH_INLINE_CONSTEXPR RAPIDHASH_INLINE
85 #ifndef RAPIDHASH_UNROLLED
86 # define RAPIDHASH_COMPACT
87 #elif defined(RAPIDHASH_COMPACT)
88 # error "cannot define RAPIDHASH_COMPACT and RAPIDHASH_UNROLLED simultaneously."
97 #ifndef RAPIDHASH_PROTECTED
98 # define RAPIDHASH_FAST
99 #elif defined(RAPIDHASH_FAST)
100 # error "cannot define RAPIDHASH_PROTECTED and RAPIDHASH_FAST simultaneously."
106 #if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
107 # define _likely_(x) __builtin_expect(x,1)
108 # define _unlikely_(x) __builtin_expect(x,0)
110 # define _likely_(x) (x)
111 # define _unlikely_(x) (x)
117 #ifndef RAPIDHASH_LITTLE_ENDIAN
118 # if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
119 # define RAPIDHASH_LITTLE_ENDIAN
120 # elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
121 # define RAPIDHASH_BIG_ENDIAN
123 # warning "could not determine endianness! Falling back to little endian."
124 # define RAPIDHASH_LITTLE_ENDIAN
131 RAPIDHASH_CONSTEXPR uint64_t rapid_secret[8] = {
132 0x2d358dccaa6c78a5ull,
133 0x8bb84b93962eacc9ull,
134 0x4b33a62ed433d4a3ull,
135 0x4d5a2da51de1aa47ull,
136 0xa0761d6478bd642full,
137 0xe7037ed1a0b428dbull,
138 0x90ed1765281c388cull,
139 0xaaaaaaaaaaaaaaaaull};
157 RAPIDHASH_INLINE_CONSTEXPR
void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT {
158 #if defined(__SIZEOF_INT128__)
159 __uint128_t r=*A; r*=*B;
160 #ifdef RAPIDHASH_PROTECTED
161 *A^=(uint64_t)r; *B^=(uint64_t)(r>>64);
163 *A=(uint64_t)r; *B=(uint64_t)(r>>64);
165 #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64))
167 #ifdef RAPIDHASH_PROTECTED
169 a=_umul128(*A,*B,&b);
172 *A=_umul128(*A,*B,B);
175 #ifdef RAPIDHASH_PROTECTED
181 uint64_t c = __umulh(*A, *B);
187 uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B;
188 uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t<rl;
189 uint64_t lo=t+(rm1<<32);
191 uint64_t hi=rh+(rm0>>32)+(rm1>>32)+c;
192 #ifdef RAPIDHASH_PROTECTED
209 RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B);
return A^B; }
214 #ifdef RAPIDHASH_LITTLE_ENDIAN
215 RAPIDHASH_INLINE uint64_t rapid_read64(
const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p,
sizeof(uint64_t));
return v;}
216 RAPIDHASH_INLINE uint64_t rapid_read32(
const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p,
sizeof(uint32_t));
return v;}
217 #elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
218 RAPIDHASH_INLINE uint64_t rapid_read64(
const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p,
sizeof(uint64_t));
return __builtin_bswap64(v);}
219 RAPIDHASH_INLINE uint64_t rapid_read32(
const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p,
sizeof(uint32_t));
return __builtin_bswap32(v);}
220 #elif defined(_MSC_VER)
221 RAPIDHASH_INLINE uint64_t rapid_read64(
const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p,
sizeof(uint64_t));
return _byteswap_uint64(v);}
222 RAPIDHASH_INLINE uint64_t rapid_read32(
const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p,
sizeof(uint32_t));
return _byteswap_ulong(v);}
224 RAPIDHASH_INLINE uint64_t rapid_read64(
const uint8_t *p) RAPIDHASH_NOEXCEPT {
225 uint64_t v; memcpy(&v, p, 8);
226 return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000));
228 RAPIDHASH_INLINE uint64_t rapid_read32(
const uint8_t *p) RAPIDHASH_NOEXCEPT {
229 uint32_t v; memcpy(&v, p, 4);
230 return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000));
244RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(
const void *key,
size_t len, uint64_t seed,
const uint64_t* secret) RAPIDHASH_NOEXCEPT {
245 const uint8_t *p=(
const uint8_t *)key;
246 seed ^= rapid_mix(seed ^ secret[2], secret[1]);
249 if (_likely_(len <= 16)) {
253 const uint8_t* plast = p + len - 8;
255 b = rapid_read64(plast);
257 const uint8_t* plast = p + len - 4;
259 b = rapid_read32(plast);
261 }
else if (len > 0) {
262 a = (((uint64_t)p[0])<<45)|p[len-1];
268 uint64_t see1 = seed, see2 = seed;
269 uint64_t see3 = seed, see4 = seed;
270 uint64_t see5 = seed, see6 = seed;
271#ifdef RAPIDHASH_COMPACT
273 seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed);
274 see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1);
275 see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2);
276 see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3);
277 see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4);
278 see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5);
279 see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6);
285 seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed);
286 see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1);
287 see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2);
288 see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3);
289 see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4);
290 see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5);
291 see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6);
292 seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed);
293 see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1);
294 see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2);
295 see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3);
296 see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4);
297 see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5);
298 see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6);
303 seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed);
304 see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1);
305 see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2);
306 see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3);
307 see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4);
308 see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5);
309 see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6);
322 seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed);
324 seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed);
326 seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed);
328 seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed);
330 seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed);
332 seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed);
339 a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8);
344 return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i);
357 RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashMicro_internal(
const void *key,
size_t len, uint64_t seed,
const uint64_t* secret) RAPIDHASH_NOEXCEPT {
358 const uint8_t *p=(
const uint8_t *)key;
359 seed ^= rapid_mix(seed ^ secret[2], secret[1]);
362 if (_likely_(len <= 16)) {
366 const uint8_t* plast = p + len - 8;
368 b = rapid_read64(plast);
370 const uint8_t* plast = p + len - 4;
372 b = rapid_read32(plast);
374 }
else if (len > 0) {
375 a = (((uint64_t)p[0])<<45)|p[len-1];
381 uint64_t see1 = seed, see2 = seed;
382 uint64_t see3 = seed, see4 = seed;
384 seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed);
385 see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1);
386 see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2);
387 see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3);
388 see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4);
398 seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed);
400 seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed);
402 seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed);
404 seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed);
409 a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8);
414 return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i);
427 RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashNano_internal(
const void *key,
size_t len, uint64_t seed,
const uint64_t* secret) RAPIDHASH_NOEXCEPT {
428 const uint8_t *p=(
const uint8_t *)key;
429 seed ^= rapid_mix(seed ^ secret[2], secret[1]);
432 if (_likely_(len <= 16)) {
436 const uint8_t* plast = p + len - 8;
438 b = rapid_read64(plast);
440 const uint8_t* plast = p + len - 4;
442 b = rapid_read32(plast);
444 }
else if (len > 0) {
445 a = (((uint64_t)p[0])<<45)|p[len-1];
451 uint64_t see1 = seed, see2 = seed;
453 seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed);
454 see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1);
455 see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2);
463 seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed);
465 seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed);
468 a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8);
473 return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i);
487RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(
const void *key,
size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT {
488 return rapidhash_internal(key, len, seed, rapid_secret);
501RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash(
const void *key,
size_t len) RAPIDHASH_NOEXCEPT {
502 return rapidhash_withSeed(key, len, 0);
520 RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashMicro_withSeed(
const void *key,
size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT {
521 return rapidhashMicro_internal(key, len, seed, rapid_secret);
534RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashMicro(
const void *key,
size_t len) RAPIDHASH_NOEXCEPT {
535 return rapidhashMicro_withSeed(key, len, 0);
549 RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashNano_withSeed(
const void *key,
size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT {
550 return rapidhashNano_internal(key, len, seed, rapid_secret);
567RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhashNano(
const void *key,
size_t len) RAPIDHASH_NOEXCEPT {
568 return rapidhashNano_withSeed(key, len, 0);