771 lines
30 KiB
C
771 lines
30 KiB
C
/*
|
|
* xxHash - Extremely Fast Hash algorithm
|
|
* Copyright (C) 2020 Yann Collet
|
|
*
|
|
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following disclaimer
|
|
* in the documentation and/or other materials provided with the
|
|
* distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* You can contact the author at:
|
|
* - xxHash homepage: https://www.xxhash.com
|
|
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
|
*/
|
|
|
|
|
|
/*!
|
|
* @file xxh_x86dispatch.c
|
|
*
|
|
* Automatic dispatcher code for the @ref xxh3_family on x86-based targets.
|
|
*
|
|
* Optional add-on.
|
|
*
|
|
* **Compile this file with the default flags for your target.** Do not compile
|
|
* with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be
|
|
* an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details.
|
|
*
|
|
* @defgroup dispatch x86 Dispatcher
|
|
* @{
|
|
*/
|
|
|
|
#if defined (__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64))
|
|
# error "Dispatching is currently only supported on x86 and x86_64."
|
|
#endif
|
|
|
|
/*!
|
|
* @def XXH_X86DISPATCH_ALLOW_AVX
|
|
* @brief Disables the AVX sanity check.
|
|
*
|
|
* Don't compile xxh_x86dispatch.c with options like `-mavx*`, `-march=native`,
|
|
* or `/arch:AVX*`. It is intended to be compiled for the minimum target, and
|
|
* it selectively enables SSE2, AVX2, and AVX512 when it is needed.
|
|
*
|
|
* Using this option _globally_ allows this feature, and therefore makes it
|
|
* undefined behavior to execute on any CPU without said feature.
|
|
*
|
|
* Even if the source code isn't directly using AVX intrinsics in a function,
|
|
* the compiler can still generate AVX code from autovectorization and by
|
|
* "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128).
|
|
*
|
|
* Use the same flags that you use to compile the rest of the program; this
|
|
* file will safely generate SSE2, AVX2, and AVX512 without these flags.
|
|
*
|
|
* Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check, and feel free to open
|
|
* an issue if there is a target in the future where AVX is a default feature.
|
|
*/
|
|
#ifdef XXH_DOXYGEN
|
|
# define XXH_X86DISPATCH_ALLOW_AVX
|
|
#endif
|
|
|
|
#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX)
|
|
# error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above."
|
|
#endif
|
|
|
|
#ifdef __has_include
|
|
# define XXH_HAS_INCLUDE(header) __has_include(header)
|
|
#else
|
|
# define XXH_HAS_INCLUDE(header) 0
|
|
#endif
|
|
|
|
/*!
|
|
* @def XXH_DISPATCH_SCALAR
|
|
* @brief Enables/dispatching the scalar code path.
|
|
*
|
|
* If this is defined to 0, SSE2 support is assumed. This reduces code size
|
|
* when the scalar path is not needed.
|
|
*
|
|
* This is automatically defined to 0 when...
|
|
* - SSE2 support is enabled in the compiler
|
|
* - Targeting x86_64
|
|
* - Targeting Android x86
|
|
* - Targeting macOS
|
|
*/
|
|
#ifndef XXH_DISPATCH_SCALAR
|
|
# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \
|
|
|| defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \
|
|
|| defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */
|
|
# define XXH_DISPATCH_SCALAR 0 /* disable */
|
|
# else
|
|
# define XXH_DISPATCH_SCALAR 1
|
|
# endif
|
|
#endif
|
|
/*!
|
|
* @def XXH_DISPATCH_AVX2
|
|
* @brief Enables/disables dispatching for AVX2.
|
|
*
|
|
* This is automatically detected if it is not defined.
|
|
* - GCC 4.7 and later are known to support AVX2, but >4.9 is required for
|
|
* to get the AVX2 intrinsics and typedefs without -mavx -mavx2.
|
|
* - Visual Studio 2013 Update 2 and later are known to support AVX2.
|
|
* - The GCC/Clang internal header `<avx2intrin.h>` is detected. While this is
|
|
* not allowed to be included directly, it still appears in the builtin
|
|
* include path and is detectable with `__has_include`.
|
|
*
|
|
* @see XXH_AVX2
|
|
*/
|
|
#ifndef XXH_DISPATCH_AVX2
|
|
# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \
|
|
|| (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \
|
|
|| (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \
|
|
|| XXH_HAS_INCLUDE(<avx2intrin.h>) /* GCC/Clang internal header */
|
|
# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */
|
|
# else
|
|
# define XXH_DISPATCH_AVX2 0
|
|
# endif
|
|
#endif /* XXH_DISPATCH_AVX2 */
|
|
|
|
/*!
|
|
* @def XXH_DISPATCH_AVX512
|
|
* @brief Enables/disables dispatching for AVX512.
|
|
*
|
|
* Automatically detected if one of the following conditions is met:
|
|
* - GCC 4.9 and later are known to support AVX512.
|
|
* - Visual Studio 2017 and later are known to support AVX2.
|
|
* - The GCC/Clang internal header `<avx512fintrin.h>` is detected. While this
|
|
* is not allowed to be included directly, it still appears in the builtin
|
|
* include path and is detectable with `__has_include`.
|
|
*
|
|
* @see XXH_AVX512
|
|
*/
|
|
#ifndef XXH_DISPATCH_AVX512
|
|
# if (defined(__GNUC__) \
|
|
&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \
|
|
|| (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \
|
|
|| XXH_HAS_INCLUDE(<avx512fintrin.h>) /* GCC/Clang internal header */
|
|
# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */
|
|
# else
|
|
# define XXH_DISPATCH_AVX512 0
|
|
# endif
|
|
#endif /* XXH_DISPATCH_AVX512 */
|
|
|
|
/*!
|
|
* @def XXH_TARGET_SSE2
|
|
* @brief Allows a function to be compiled with SSE2 intrinsics.
|
|
*
|
|
* Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used
|
|
* even with `-mno-sse2`.
|
|
*
|
|
* @def XXH_TARGET_AVX2
|
|
* @brief Like @ref XXH_TARGET_SSE2, but for AVX2.
|
|
*
|
|
* @def XXH_TARGET_AVX512
|
|
* @brief Like @ref XXH_TARGET_SSE2, but for AVX512.
|
|
*/
|
|
#if defined(__GNUC__)
|
|
# include <emmintrin.h> /* SSE2 */
|
|
# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
# include <immintrin.h> /* AVX2, AVX512F */
|
|
# endif
|
|
# define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
|
|
# define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
|
|
# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
|
|
#elif defined(_MSC_VER)
|
|
# include <intrin.h>
|
|
# define XXH_TARGET_SSE2
|
|
# define XXH_TARGET_AVX2
|
|
# define XXH_TARGET_AVX512
|
|
#else
|
|
# error "Dispatching is currently not supported for your compiler."
|
|
#endif
|
|
|
|
#ifdef XXH_DISPATCH_DEBUG
|
|
/* debug logging */
|
|
# include <stdio.h>
|
|
# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); }
|
|
#else
|
|
# define XXH_debugPrint(str) ((void)0)
|
|
# undef NDEBUG /* avoid redefinition */
|
|
# define NDEBUG
|
|
#endif
|
|
#include <assert.h>
|
|
|
|
#define XXH_INLINE_ALL
|
|
#define XXH_X86DISPATCH
|
|
#include "xxhash.h"
|
|
|
|
/*
|
|
* Support both AT&T and Intel dialects
|
|
*
|
|
* GCC doesn't convert AT&T syntax to Intel syntax, and will error out if
|
|
* compiled with -masm=intel. Instead, it supports dialect switching with
|
|
* curly braces: { AT&T syntax | Intel syntax }
|
|
*
|
|
* Clang's integrated assembler automatically converts AT&T syntax to Intel if
|
|
* needed, making the dialect switching useless (it isn't even supported).
|
|
*
|
|
* Note: Comments are written in the inline assembly itself.
|
|
*/
|
|
#ifdef __clang__
|
|
# define XXH_I_ATT(intel, att) att "\n\t"
|
|
#else
|
|
# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t"
|
|
#endif
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Runs CPUID.
|
|
*
|
|
* @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively.
|
|
* @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }`
|
|
*/
|
|
static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
|
|
{
|
|
#if defined(_MSC_VER)
|
|
__cpuidex(abcd, eax, ecx);
|
|
#else
|
|
xxh_u32 ebx, edx;
|
|
# if defined(__i386__) && defined(__PIC__)
|
|
__asm__(
|
|
"# Call CPUID\n\t"
|
|
"#\n\t"
|
|
"# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t"
|
|
"# EBX, so we use EDI instead.\n\t"
|
|
XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi")
|
|
XXH_I_ATT("cpuid", "cpuid" )
|
|
XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi")
|
|
: "=D" (ebx),
|
|
# else
|
|
__asm__(
|
|
"# Call CPUID\n\t"
|
|
XXH_I_ATT("cpuid", "cpuid")
|
|
: "=b" (ebx),
|
|
# endif
|
|
"+a" (eax), "+c" (ecx), "=d" (edx));
|
|
abcd[0] = eax;
|
|
abcd[1] = ebx;
|
|
abcd[2] = ecx;
|
|
abcd[3] = edx;
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Modified version of Intel's guide
|
|
* https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family
|
|
*/
|
|
|
|
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
/*!
|
|
* @internal
|
|
* @brief Runs `XGETBV`.
|
|
*
|
|
* While the CPU may support AVX2, the operating system might not properly save
|
|
* the full YMM/ZMM registers.
|
|
*
|
|
* xgetbv is used for detecting this: Any compliant operating system will define
|
|
* a set of flags in the xcr0 register indicating how it saves the AVX registers.
|
|
*
|
|
* You can manually disable this flag on Windows by running, as admin:
|
|
*
|
|
* bcdedit.exe /set xsavedisable 1
|
|
*
|
|
* and rebooting. Run the same command with 0 to re-enable it.
|
|
*/
|
|
static xxh_u64 XXH_xgetbv(void)
|
|
{
|
|
#if defined(_MSC_VER)
|
|
return _xgetbv(0); /* min VS2010 SP1 compiler is required */
|
|
#else
|
|
xxh_u32 xcr0_lo, xcr0_hi;
|
|
__asm__(
|
|
"# Call XGETBV\n\t"
|
|
"#\n\t"
|
|
"# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t"
|
|
"# the XGETBV opcode, so we encode it by hand instead.\n\t"
|
|
"# See <https://github.com/asmjit/asmjit/issues/78> for details.\n\t"
|
|
".byte 0x0f, 0x01, 0xd0\n\t"
|
|
: "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0));
|
|
return xcr0_lo | ((xxh_u64)xcr0_hi << 32);
|
|
#endif
|
|
}
|
|
#endif
|
|
|
|
#define XXH_SSE2_CPUID_MASK (1 << 26)
|
|
#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27))
|
|
#define XXH_AVX2_CPUID_MASK (1 << 5)
|
|
#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1))
|
|
#define XXH_AVX512F_CPUID_MASK (1 << 16)
|
|
#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1))
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Returns the best XXH3 implementation.
|
|
*
|
|
* Runs various CPUID/XGETBV tests to try and determine the best implementation.
|
|
*
|
|
* @ret The best @ref XXH_VECTOR implementation.
|
|
* @see XXH_VECTOR_TYPES
|
|
*/
|
|
static int XXH_featureTest(void)
|
|
{
|
|
xxh_u32 abcd[4];
|
|
xxh_u32 max_leaves;
|
|
int best = XXH_SCALAR;
|
|
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
xxh_u64 xgetbv_val;
|
|
#endif
|
|
#if defined(__GNUC__) && defined(__i386__)
|
|
xxh_u32 cpuid_supported;
|
|
__asm__(
|
|
"# For the sake of ruthless backwards compatibility, check if CPUID\n\t"
|
|
"# is supported in the EFLAGS on i386.\n\t"
|
|
"# This is not necessary on x86_64 - CPUID is mandatory.\n\t"
|
|
"# The ID flag (bit 21) in the EFLAGS register indicates support\n\t"
|
|
"# for the CPUID instruction. If a software procedure can set and\n\t"
|
|
"# clear this flag, the processor executing the procedure supports\n\t"
|
|
"# the CPUID instruction.\n\t"
|
|
"# <https://c9x.me/x86/html/file_module_x86_id_45.html>\n\t"
|
|
"#\n\t"
|
|
"# Routine is from <https://wiki.osdev.org/CPUID>.\n\t"
|
|
|
|
"# Save EFLAGS\n\t"
|
|
XXH_I_ATT("pushfd", "pushfl" )
|
|
"# Store EFLAGS\n\t"
|
|
XXH_I_ATT("pushfd", "pushfl" )
|
|
"# Invert the ID bit in stored EFLAGS\n\t"
|
|
XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)")
|
|
"# Load stored EFLAGS (with ID bit inverted)\n\t"
|
|
XXH_I_ATT("popfd", "popfl" )
|
|
"# Store EFLAGS again (ID bit may or not be inverted)\n\t"
|
|
XXH_I_ATT("pushfd", "pushfl" )
|
|
"# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t"
|
|
XXH_I_ATT("pop eax", "popl %%eax" )
|
|
"# eax = whichever bits were changed\n\t"
|
|
XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" )
|
|
"# Restore original EFLAGS\n\t"
|
|
XXH_I_ATT("popfd", "popfl" )
|
|
"# eax = zero if ID bit can't be changed, else non-zero\n\t"
|
|
XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" )
|
|
: "=a" (cpuid_supported) :: "cc");
|
|
|
|
if (XXH_unlikely(!cpuid_supported)) {
|
|
XXH_debugPrint("CPUID support is not detected!");
|
|
return best;
|
|
}
|
|
|
|
#endif
|
|
/* Check how many CPUID pages we have */
|
|
XXH_cpuid(0, 0, abcd);
|
|
max_leaves = abcd[0];
|
|
|
|
/* Shouldn't happen on hardware, but happens on some QEMU configs. */
|
|
if (XXH_unlikely(max_leaves == 0)) {
|
|
XXH_debugPrint("Max CPUID leaves == 0!");
|
|
return best;
|
|
}
|
|
|
|
/* Check for SSE2, OSXSAVE and xgetbv */
|
|
XXH_cpuid(1, 0, abcd);
|
|
|
|
/*
|
|
* Test for SSE2. The check is redundant on x86_64, but it doesn't hurt.
|
|
*/
|
|
if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK))
|
|
return best;
|
|
|
|
XXH_debugPrint("SSE2 support detected.");
|
|
|
|
best = XXH_SSE2;
|
|
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
|
|
/* Make sure we have enough leaves */
|
|
if (XXH_unlikely(max_leaves < 7))
|
|
return best;
|
|
|
|
/* Test for OSXSAVE and XGETBV */
|
|
if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK)
|
|
return best;
|
|
|
|
/* CPUID check for AVX features */
|
|
XXH_cpuid(7, 0, abcd);
|
|
|
|
xgetbv_val = XXH_xgetbv();
|
|
#if XXH_DISPATCH_AVX2
|
|
/* Validate that AVX2 is supported by the CPU */
|
|
if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK)
|
|
return best;
|
|
|
|
/* Validate that the OS supports YMM registers */
|
|
if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) {
|
|
XXH_debugPrint("AVX2 supported by the CPU, but not the OS.");
|
|
return best;
|
|
}
|
|
|
|
/* AVX2 supported */
|
|
XXH_debugPrint("AVX2 support detected.");
|
|
best = XXH_AVX2;
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
/* Check if AVX512F is supported by the CPU */
|
|
if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) {
|
|
XXH_debugPrint("AVX512F not supported by CPU");
|
|
return best;
|
|
}
|
|
|
|
/* Validate that the OS supports ZMM registers */
|
|
if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) {
|
|
XXH_debugPrint("AVX512F supported by the CPU, but not the OS.");
|
|
return best;
|
|
}
|
|
|
|
/* AVX512F supported */
|
|
XXH_debugPrint("AVX512F support detected.");
|
|
best = XXH_AVX512;
|
|
#endif
|
|
#endif
|
|
return best;
|
|
}
|
|
|
|
|
|
/* === Vector implementations === */
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Defines the various dispatch functions.
|
|
*
|
|
* TODO: Consolidate?
|
|
*
|
|
* @param suffix The suffix for the functions, e.g. sse2 or scalar
|
|
* @param target XXH_TARGET_* or empty.
|
|
*/
|
|
#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \
|
|
\
|
|
/* === XXH3, default variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH64_hash_t \
|
|
XXHL64_default_##suffix(const void* XXH_RESTRICT input, size_t len) \
|
|
{ \
|
|
return XXH3_hashLong_64b_internal( \
|
|
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
|
|
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH3, Seeded variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH64_hash_t \
|
|
XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
|
XXH64_hash_t seed) \
|
|
{ \
|
|
return XXH3_hashLong_64b_withSeed_internal( \
|
|
input, len, seed, XXH3_accumulate_512_##suffix, \
|
|
XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH3, Secret variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH64_hash_t \
|
|
XXHL64_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
|
const void* secret, size_t secretLen) \
|
|
{ \
|
|
return XXH3_hashLong_64b_internal( \
|
|
input, len, secret, secretLen, \
|
|
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH3 update variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH_errorcode \
|
|
XXH3_update_##suffix(XXH3_state_t* state, const void* input, size_t len) \
|
|
{ \
|
|
return XXH3_update(state, (const xxh_u8*)input, len, \
|
|
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \
|
|
} \
|
|
\
|
|
/* === XXH128 default variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH128_hash_t \
|
|
XXHL128_default_##suffix(const void* XXH_RESTRICT input, size_t len) \
|
|
{ \
|
|
return XXH3_hashLong_128b_internal( \
|
|
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
|
|
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
|
|
); \
|
|
} \
|
|
\
|
|
/* === XXH128 Secret variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH128_hash_t \
|
|
XXHL128_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
|
const void* XXH_RESTRICT secret, size_t secretLen) \
|
|
{ \
|
|
return XXH3_hashLong_128b_internal( \
|
|
input, len, (const xxh_u8*)secret, secretLen, \
|
|
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \
|
|
} \
|
|
\
|
|
/* === XXH128 Seeded variants === */ \
|
|
\
|
|
XXH_NO_INLINE target XXH128_hash_t \
|
|
XXHL128_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \
|
|
XXH64_hash_t seed) \
|
|
{ \
|
|
return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \
|
|
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix, \
|
|
XXH3_initCustomSecret_##suffix); \
|
|
}
|
|
|
|
/* End XXH_DEFINE_DISPATCH_FUNCS */
|
|
|
|
#if XXH_DISPATCH_SCALAR
|
|
XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */)
|
|
#endif
|
|
XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2)
|
|
#if XXH_DISPATCH_AVX2
|
|
XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2)
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512)
|
|
#endif
|
|
#undef XXH_DEFINE_DISPATCH_FUNCS
|
|
|
|
/* ==== Dispatchers ==== */
|
|
|
|
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t);
|
|
|
|
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH3_state_t*, const void*, size_t);
|
|
|
|
typedef struct {
|
|
XXH3_dispatchx86_hashLong64_default hashLong64_default;
|
|
XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed;
|
|
XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret;
|
|
XXH3_dispatchx86_update update;
|
|
} XXH_dispatchFunctions_s;
|
|
|
|
#define XXH_NB_DISPATCHES 4
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Table of dispatchers for @ref XXH3_64bits().
|
|
*
|
|
* @pre The indices must match @ref XXH_VECTOR_TYPE.
|
|
*/
|
|
static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = {
|
|
#if XXH_DISPATCH_SCALAR
|
|
/* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar },
|
|
#else
|
|
/* Scalar */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
/* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 },
|
|
#if XXH_DISPATCH_AVX2
|
|
/* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 },
|
|
#else
|
|
/* AVX2 */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
/* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 }
|
|
#else
|
|
/* AVX512 */ { NULL, NULL, NULL, NULL }
|
|
#endif
|
|
};
|
|
/*!
|
|
* @internal
|
|
* @brief The selected dispatch table for @ref XXH3_64bits().
|
|
*/
|
|
static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL };
|
|
|
|
|
|
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t);
|
|
|
|
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
|
|
|
|
typedef struct {
|
|
XXH3_dispatchx86_hashLong128_default hashLong128_default;
|
|
XXH3_dispatchx86_hashLong128_withSeed hashLong128_seed;
|
|
XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret;
|
|
XXH3_dispatchx86_update update;
|
|
} XXH_dispatch128Functions_s;
|
|
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Table of dispatchers for @ref XXH3_128bits().
|
|
*
|
|
* @pre The indices must match @ref XXH_VECTOR_TYPE.
|
|
*/
|
|
static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = {
|
|
#if XXH_DISPATCH_SCALAR
|
|
/* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar },
|
|
#else
|
|
/* Scalar */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
/* SSE2 */ { XXHL128_default_sse2, XXHL128_seed_sse2, XXHL128_secret_sse2, XXH3_update_sse2 },
|
|
#if XXH_DISPATCH_AVX2
|
|
/* AVX2 */ { XXHL128_default_avx2, XXHL128_seed_avx2, XXHL128_secret_avx2, XXH3_update_avx2 },
|
|
#else
|
|
/* AVX2 */ { NULL, NULL, NULL, NULL },
|
|
#endif
|
|
#if XXH_DISPATCH_AVX512
|
|
/* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 }
|
|
#else
|
|
/* AVX512 */ { NULL, NULL, NULL, NULL }
|
|
#endif
|
|
};
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief The selected dispatch table for @ref XXH3_64bits().
|
|
*/
|
|
static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL };
|
|
|
|
/*!
|
|
* @internal
|
|
* @brief Runs a CPUID check and sets the correct dispatch tables.
|
|
*/
|
|
static void XXH_setDispatch(void)
|
|
{
|
|
int vecID = XXH_featureTest();
|
|
XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1);
|
|
assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512);
|
|
#if !XXH_DISPATCH_SCALAR
|
|
assert(vecID != XXH_SCALAR);
|
|
#endif
|
|
#if !XXH_DISPATCH_AVX512
|
|
assert(vecID != XXH_AVX512);
|
|
#endif
|
|
#if !XXH_DISPATCH_AVX2
|
|
assert(vecID != XXH_AVX2);
|
|
#endif
|
|
XXH_g_dispatch = XXH_kDispatch[vecID];
|
|
XXH_g_dispatch128 = XXH_kDispatch128[vecID];
|
|
}
|
|
|
|
|
|
/* ==== XXH3 public functions ==== */
|
|
|
|
static XXH64_hash_t
|
|
XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
|
{
|
|
(void)seed64; (void)secret; (void)secretLen;
|
|
if (XXH_g_dispatch.hashLong64_default == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch.hashLong64_default(input, len);
|
|
}
|
|
|
|
XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len)
|
|
{
|
|
return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection);
|
|
}
|
|
|
|
static XXH64_hash_t
|
|
XXH3_hashLong_64b_withSeed_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
|
{
|
|
(void)secret; (void)secretLen;
|
|
if (XXH_g_dispatch.hashLong64_seed == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch.hashLong64_seed(input, len, seed64);
|
|
}
|
|
|
|
XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed)
|
|
{
|
|
return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection);
|
|
}
|
|
|
|
static XXH64_hash_t
|
|
XXH3_hashLong_64b_withSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
|
|
{
|
|
(void)seed64;
|
|
if (XXH_g_dispatch.hashLong64_secret == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen);
|
|
}
|
|
|
|
XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen)
|
|
{
|
|
return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection);
|
|
}
|
|
|
|
XXH_errorcode
|
|
XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len)
|
|
{
|
|
if (XXH_g_dispatch.update == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch.update(state, (const xxh_u8*)input, len);
|
|
}
|
|
|
|
|
|
/* ==== XXH128 public functions ==== */
|
|
|
|
static XXH128_hash_t
|
|
XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
{
|
|
(void)seed64; (void)secret; (void)secretLen;
|
|
if (XXH_g_dispatch128.hashLong128_default == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch128.hashLong128_default(input, len);
|
|
}
|
|
|
|
XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len)
|
|
{
|
|
return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection);
|
|
}
|
|
|
|
static XXH128_hash_t
|
|
XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
{
|
|
(void)secret; (void)secretLen;
|
|
if (XXH_g_dispatch128.hashLong128_seed == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch128.hashLong128_seed(input, len, seed64);
|
|
}
|
|
|
|
XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed)
|
|
{
|
|
return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection);
|
|
}
|
|
|
|
static XXH128_hash_t
|
|
XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len,
|
|
XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
{
|
|
(void)seed64;
|
|
if (XXH_g_dispatch128.hashLong128_secret == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen);
|
|
}
|
|
|
|
XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen)
|
|
{
|
|
return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection);
|
|
}
|
|
|
|
XXH_errorcode
|
|
XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len)
|
|
{
|
|
if (XXH_g_dispatch128.update == NULL) XXH_setDispatch();
|
|
return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len);
|
|
}
|
|
|
|
#if defined (__cplusplus)
|
|
}
|
|
#endif
|
|
/*! @} */
|