From f96a5e42cf7b99253cd0ff3eff619ce11e0b9888 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Wed, 20 May 2026 20:13:14 +0200 Subject: [PATCH 1/9] feat: add Windows ARM64 (MSVC) build support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enables `pip install .` to produce a native ARM64 Python wheel when building on Windows ARM64 with MSVC + Visual Studio Build Tools 2022. Four small changes cover the gaps: * src/ailego/CMakeLists.txt — the arm/arm64 branch of the AUTO_DETECT_ARCH block had `if(MSVC) return() endif()`, which bailed out of the file before `cc_library(zvec_ailego ...)` was reached. Downstream (`src/binding/c`, `src/core/*`, `src/db/*`, tests, etc.) then failed to configure with "No target zvec_ailego". The NEON source glob + `-march=armv8-a` flag setup is still only useful for GCC/Clang, so wrap just that block in `if(NOT MSVC)` and let the target definition proceed on MSVC ARM64. * src/ailego/internal/cpu_features.cc — the MSVC branch unconditionally used `__cpuidex`, which is x86/x64-only; the GCC branch used `__get_cpuid`, gated only by `!defined(__ARM_ARCH)`. On MSVC ARM64 neither macro applies and the build failed with "__cpuidex: identifier not found". Scope the two branches to x86/x64 explicitly so MSVC ARM64 falls through to the empty-stub constructor that already exists for non-x86 targets. * thirdparty/arrow/CMakeLists.txt — Arrow 21.0 ships xsimd 13.0, which does not implement `xsimd::make_sized_batch_t` for MSVC ARM64. Pass `-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE` only when `CMAKE_SYSTEM_PROCESSOR` is ARM64, so x64 MSVC keeps its SSE4.2 path unchanged. * thirdparty/arrow/arrow.windows-arm64.patch (new) — Arrow's vendored PCG header (`arrow/vendored/pcg/pcg_uint128.hpp`) uses an x86-only endianness check and calls `_umul128`, which is not available on MSVC ARM64 (the equivalent intrinsic is `__umulh`). Add `_M_ARM64`, `_M_ARM`, `__aarch64__`, and `__arm__` to the little-endian case, branch to `__umulh` on ARM, and guard `#pragma intrinsic(_umul128)` so it is not referenced on ARM. Applied via the existing `apply_patch_once` mechanism, scoped to MSVC+ARM64. Tested on Windows 11 ARM64 (Surface class hardware) with MSVC 14.44 and VS Build Tools 2022 component `Microsoft.VisualStudio.Component.VC.Tools.ARM64` installed. `pip install .` produces a working `zvec` wheel and the Python extension imports cleanly (`zvec: OK (0.3.2.dev2)`). --- src/ailego/CMakeLists.txt | 43 +++++++++++----------- src/ailego/internal/cpu_features.cc | 10 ++--- thirdparty/arrow/CMakeLists.txt | 12 ++++++ thirdparty/arrow/arrow.windows-arm64.patch | 35 ++++++++++++++++++ 4 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 thirdparty/arrow/arrow.windows-arm64.patch diff --git a/src/ailego/CMakeLists.txt b/src/ailego/CMakeLists.txt index 29cf22cd1..444b5e48b 100644 --- a/src/ailego/CMakeLists.txt +++ b/src/ailego/CMakeLists.txt @@ -91,29 +91,28 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) ) endforeach() elseif (HOST_ARCH MATCHES "^(arm|arm64)$") - if(MSVC) - return() - endif() - set(MATH_MARCH_FLAG_NEON "-march=armv8-a") - - file(GLOB_RECURSE MATH_FILES_NEON - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.c - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.c - ) + if(NOT MSVC) + set(MATH_MARCH_FLAG_NEON "-march=armv8-a") + + file(GLOB_RECURSE MATH_FILES_NEON + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.c + ) - foreach(MATH_FILE ${MATH_FILES_NEON}) - set_source_files_properties( - ${MATH_FILE} - PROPERTIES - COMPILE_FLAGS "${MATH_MARCH_FLAG_NEON}" - ) - endforeach() + foreach(MATH_FILE ${MATH_FILES_NEON}) + set_source_files_properties( + ${MATH_FILE} + PROPERTIES + COMPILE_FLAGS "${MATH_MARCH_FLAG_NEON}" + ) + endforeach() + endif() endif() endif() diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 066b0a6a3..ef8a49cfc 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -15,11 +15,9 @@ #include "cpu_features.h" #include -#if defined(_MSC_VER) +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) #include -#endif - -#if (defined(__x86_64__) || defined(__i386__)) && !defined(_MSC_VER) +#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(__aarch64__) #include #endif @@ -36,7 +34,7 @@ namespace internal { CpuFeatures::CpuFlags CpuFeatures::flags_; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) CpuFeatures::CpuFlags::CpuFlags(void) : L1_ECX(0), L1_EDX(0), L7_EBX(0), L7_ECX(0), L7_EDX(0) { int l1[4] = {0, 0, 0, 0}; @@ -50,7 +48,7 @@ CpuFeatures::CpuFlags::CpuFlags(void) L7_ECX = l7[2]; L7_EDX = l7[3]; } -#elif defined(__x86_64__) || defined(__i386__) +#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(__aarch64__) CpuFeatures::CpuFlags::CpuFlags(void) : L1_ECX(0), L1_EDX(0), L7_EBX(0), L7_ECX(0), L7_EDX(0) { uint32_t eax, ebx, ecx, edx; diff --git a/thirdparty/arrow/CMakeLists.txt b/thirdparty/arrow/CMakeLists.txt index d6d926db7..f8da1e851 100644 --- a/thirdparty/arrow/CMakeLists.txt +++ b/thirdparty/arrow/CMakeLists.txt @@ -12,6 +12,10 @@ endif() if(MSVC) set(ARROW_WIN_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.windows.patch) apply_patch_once("arrow_windows_crt_fix" "${ARROW_SRC_DIR}" "${ARROW_WIN_PATCH}") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ARM64|arm64|aarch64)$") + set(ARROW_WIN_ARM64_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.windows-arm64.patch) + apply_patch_once("arrow_windows_arm64_fix" "${ARROW_SRC_DIR}" "${ARROW_WIN_ARM64_PATCH}") + endif() endif() include(ExternalProject) @@ -129,6 +133,14 @@ elseif (MSVC) -DARROW_USE_STATIC_CRT=${ZVEC_USE_STATIC_CRT} "-DCMAKE_MSVC_RUNTIME_LIBRARY=${_ARROW_MSVC_RUNTIME}" ) + # Arrow 21.0's xsimd-13 does not provide make_sized_batch_t for MSVC ARM64, + # so disable SIMD on that target. x86/x64 MSVC keeps the default SSE4.2 path. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ARM64|arm64|aarch64)$") + list(APPEND ARROW_EXTRA_CMAKE_ARGS + -DARROW_SIMD_LEVEL=NONE + -DARROW_RUNTIME_SIMD_LEVEL=NONE + ) + endif() ExternalProject_Add( ARROW.BUILD PREFIX arrow SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 diff --git a/thirdparty/arrow/arrow.windows-arm64.patch b/thirdparty/arrow/arrow.windows-arm64.patch new file mode 100644 index 000000000..f815a0c49 --- /dev/null +++ b/thirdparty/arrow/arrow.windows-arm64.patch @@ -0,0 +1,35 @@ +diff --git a/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp b/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp +index 0181e69e4e..349e3b6bfa 100644 +--- a/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp ++++ b/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp +@@ -67,7 +67,8 @@ + #define PCG_LITTLE_ENDIAN 1 + #elif __BIG_ENDIAN__ || _BIG_ENDIAN + #define PCG_LITTLE_ENDIAN 0 +- #elif __x86_64 || __x86_64__ || _M_X64 || __i386 || __i386__ || _M_IX86 ++ #elif __x86_64 || __x86_64__ || _M_X64 || __i386 || __i386__ || _M_IX86 \ ++ || _M_ARM64 || _M_ARM || __aarch64__ || __arm__ + #define PCG_LITTLE_ENDIAN 1 + #elif __powerpc__ || __POWERPC__ || __ppc__ || __PPC__ \ + || __m68k__ || __mc68000__ +@@ -733,7 +734,7 @@ uint_x4 operator*(const uint_x4& a, + } + + #if PCG_64BIT_SPECIALIZATIONS +-#if defined(_MSC_VER) ++#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_ARM) + #pragma intrinsic(_umul128) + #endif + +@@ -742,7 +743,10 @@ template + uint_x4 operator*(const uint_x4& a, + const uint_x4& b) + { +-#if defined(_MSC_VER) ++#if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM)) ++ uint64_t lo = a.d.v01 * b.d.v01; ++ uint64_t hi = __umulh(a.d.v01, b.d.v01); ++#elif defined(_MSC_VER) + uint64_t hi; + uint64_t lo = _umul128(a.d.v01, b.d.v01, &hi); + #else From 2d22027ecb4cf6f1577af0104679c4c38dd24f87 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Wed, 20 May 2026 20:13:14 +0200 Subject: [PATCH 2/9] ci(windows): add Windows ARM64 runner to the build matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the existing Windows job in 05-windows-build.yml to also cover `windows-11-arm` so the MSVC ARM64 build path is exercised on every PR, per request from @feihongxu0824 on #352. Changes: * Add a third row to the matrix with `platform: windows-11-arm`, `msvc_arch: arm64`, and `python_version: '3.11'` (Python 3.10 has no official Windows-on-ARM installer; 3.11 is the first). * Parameterize the existing `ilammy/msvc-dev-cmd@v1` step on `matrix.msvc_arch` instead of hard-coded `x64`, and the `actions/setup-python@v6` step on `matrix.python_version`. No changes to the x64 rows (still Python 3.10 + MSVC x64) and no changes to the build/test steps themselves — same `pip install -v .`, same C++ unittest run, same pytest, same examples. `fail-fast: false` was already set so an ARM64 regression will not hide x64 regressions and vice versa. --- .github/workflows/05-windows-build.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/05-windows-build.yml b/.github/workflows/05-windows-build.yml index dcdbda3ab..a9052aaf2 100644 --- a/.github/workflows/05-windows-build.yml +++ b/.github/workflows/05-windows-build.yml @@ -18,7 +18,16 @@ jobs: matrix: include: - platform: windows-2022 + msvc_arch: x64 + python_version: '3.10' - platform: windows-2025 + msvc_arch: x64 + python_version: '3.10' + # Windows ARM64: Python 3.10 has no official ARM64 installer; + # 3.11 is the first CPython release with a Windows-on-ARM build. + - platform: windows-11-arm + msvc_arch: arm64 + python_version: '3.11' env: SCCACHE_GHA_ENABLED: "true" @@ -47,14 +56,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.10' + python-version: ${{ matrix.python_version }} cache: 'pip' cache-dependency-path: 'pyproject.toml' - name: Set up MSVC environment uses: ilammy/msvc-dev-cmd@v1.13.0 with: - arch: x64 + arch: ${{ matrix.msvc_arch }} - name: Set up environment variables run: | From 36606a38a64811897308899eab36fb1f7372c8ed Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Wed, 20 May 2026 20:23:24 +0200 Subject: [PATCH 3/9] feat(ailego): enable NEON math kernels on MSVC ARM64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses @feihongxu0824's review comment on #352. Previously the *_neon.cc files compiled into empty translation units on MSVC ARM64 because their guards (`__ARM_NEON`, `__aarch64__`) are GCC/Clang-only macros, leaving zvec to use the scalar fallback. That fallback hits ~1 ULP precision drift versus the NEON path, which surfaced as two HnswStreamerTest cosine failures and a `NormMatrix.Norm1_General` failure on the `windows-11-arm` CI runner. Changes: * Expand `defined(__ARM_NEON)` -> `(defined(__ARM_NEON) || defined(_M_ARM64))` at the 51 source sites that gate ARM NEON kernels (math, math_batch, utility, normalizer, platform headers, dispatch tables, version probe). * Expand `defined(__aarch64__)` -> `(defined(__aarch64__) || defined(_M_ARM64))` at the 26 sites that distinguish AArch64 from ARMv7 NEON — MSVC ARM64 is AArch64 but does not predefine `__aarch64__`. As a side effect the ARMv7-only polyfills (`vaddvq_f32`/`vaddvq_s32` shims in `distance_matrix_accum_fp32.i`, `distance_matrix_fp32.i`) are correctly skipped under MSVC ARM64, where those intrinsics are built in. * `src/include/zvec/ailego/internal/platform.h`: include `` on the MSVC branch when `_M_ARM64` is defined (it was previously gated behind `!_MSC_VER`, so MSVC ARM64 saw no NEON types). * `src/ailego/CMakeLists.txt`: keep the existing `if(NOT MSVC)` wrapper around the GCC-only `-march=armv8-a` flag, and add an explicit `else()` branch with a comment explaining MSVC ARM64 does not need `-march` (NEON is the ARMv8 baseline on MSVC and the kernels are now picked up via the ALL_SRCS glob with the macro guards above). This supersedes the earlier `test(hnsw): skip two cosine self-match tests on MSVC ARM64` commit (which has been dropped from the branch). The NEON math kernels now use the same precision path as Linux/macOS ARM64, so those tests should pass natively on `windows-11-arm`. --- src/ailego/CMakeLists.txt | 7 +++++++ src/ailego/internal/cpu_features.cc | 8 +++++--- src/ailego/math/distance_matrix_accum_fp32.i | 4 ++-- src/ailego/math/distance_matrix_fp32.i | 2 +- ...euclidean_distance_matrix_fp32_dispatch.cc | 4 ++-- .../euclidean_distance_matrix_fp32_neon.cc | 2 +- .../inner_product_matrix_fp32_dispatch.cc | 6 +++--- .../math/inner_product_matrix_fp32_neon.cc | 2 +- ...euclidean_distance_matrix_fp32_dispatch.cc | 4 ++-- ...ips_euclidean_distance_matrix_fp32_neon.cc | 2 +- src/ailego/math/norm1_matrix.h | 5 ++++- src/ailego/math/norm1_matrix_fp16.cc | 3 +++ src/ailego/math/norm1_matrix_fp32.cc | 5 +++-- src/ailego/math/norm2_matrix.h | 5 ++++- src/ailego/math/norm2_matrix_fp16.cc | 2 ++ src/ailego/math/norm2_matrix_fp32.cc | 7 ++++--- src/ailego/math/normalizer.cc | 18 ++++++++++++++--- src/ailego/math/normalizer.h | 5 ++++- src/ailego/utility/bitset_helper.cc | 5 +++-- src/ailego/utility/float_helper.cc | 5 ++++- src/ailego/version.i | 2 +- .../zvec/ailego/buffer/concurrentqueue.h | 20 ++++++++++--------- src/include/zvec/ailego/internal/platform.h | 19 ++++++++++++------ .../zvec/ailego/utility/float_helper.h | 5 ++++- thirdparty/FastPFOR/CMakeLists.txt | 9 +++++++++ 25 files changed, 109 insertions(+), 47 deletions(-) diff --git a/src/ailego/CMakeLists.txt b/src/ailego/CMakeLists.txt index 444b5e48b..606b0b5e9 100644 --- a/src/ailego/CMakeLists.txt +++ b/src/ailego/CMakeLists.txt @@ -112,6 +112,13 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) COMPILE_FLAGS "${MATH_MARCH_FLAG_NEON}" ) endforeach() + else() + # MSVC on ARM64: NEON is the ARMv8 baseline and is always enabled, + # so no `-march` flag is required (MSVC does not accept GCC-style + # `-march=` anyway). The NEON math kernels still get compiled via + # the ALL_SRCS glob above; their `#if defined(__ARM_NEON)` guards + # were extended in this PR to also accept `_M_ARM64` so the bodies + # actually emit code under MSVC. endif() endif() endif() diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index ef8a49cfc..395e6fc13 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -17,7 +17,8 @@ #if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) #include -#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(__aarch64__) +#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && \ + !(defined(__aarch64__) || defined(_M_ARM64)) #include #endif @@ -48,7 +49,8 @@ CpuFeatures::CpuFlags::CpuFlags(void) L7_ECX = l7[2]; L7_EDX = l7[3]; } -#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(__aarch64__) +#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && \ + !(defined(__aarch64__) || defined(_M_ARM64)) CpuFeatures::CpuFlags::CpuFlags(void) : L1_ECX(0), L1_EDX(0), L7_EBX(0), L7_ECX(0), L7_EDX(0) { uint32_t eax, ebx, ecx, edx; @@ -336,7 +338,7 @@ bool CpuFeatures::HYPERVISOR(void) { const char *CpuFeatures::Intrinsics(void) { return "" -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) "Neon" #if defined(__ARM_FEATURE_CRC32) "+CRC" diff --git a/src/ailego/math/distance_matrix_accum_fp32.i b/src/ailego/math/distance_matrix_accum_fp32.i index c186492c6..913784076 100644 --- a/src/ailego/math/distance_matrix_accum_fp32.i +++ b/src/ailego/math/distance_matrix_accum_fp32.i @@ -30,7 +30,7 @@ _mm512_castps_si512(b))) #endif // __AVX512DQ__ -#if defined(__ARM_NEON) && !defined(__aarch64__) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) && !(defined(__aarch64__) || defined(_M_ARM64)) static inline float32_t vaddvq_f32(float32x4_t v) { float32x2_t s = vadd_f32(vget_low_f32(v), vget_high_f32(v)); return vget_lane_f32(vpadd_f32(s, s), 0); @@ -42,7 +42,7 @@ static inline int32_t vaddvq_s32(int32x4_t v) { } #endif //__ARM_NEON && !__aarch64__ -#if defined(__aarch64__) +#if (defined(__aarch64__) || defined(_M_ARM64)) #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A64 #else #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A32 diff --git a/src/ailego/math/distance_matrix_fp32.i b/src/ailego/math/distance_matrix_fp32.i index a9ddcd075..f4e6a16ab 100644 --- a/src/ailego/math/distance_matrix_fp32.i +++ b/src/ailego/math/distance_matrix_fp32.i @@ -26,7 +26,7 @@ _mm256_insertf128_ps(_mm256_castps128_ps256(b), (a), 1) #endif // __AVX__ -#if defined(__ARM_NEON) && !defined(__aarch64__) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) && !(defined(__aarch64__) || defined(_M_ARM64)) #define vdupq_laneq_f32(a, b) vdupq_n_f32(vgetq_lane_f32(a, b)) #endif // __ARM_NEON && __aarch64__ diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc index cc3044389..f0650a08e 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc @@ -18,7 +18,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, size_t size, float *out); #endif @@ -49,7 +49,7 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) SquaredEuclideanDistanceFp32NEON(m, q, dim, out); #else #if defined(__AVX512F__) diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc index aa1694e21..14ce90767 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) //! Squared Euclidean Distance void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, size_t size, float *out) { diff --git a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc index 8b289b6e6..32540296b 100644 --- a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc +++ b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc @@ -20,7 +20,7 @@ namespace ailego { //-------------------------------------------------- // Dense //-------------------------------------------------- -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) float InnerProductFp32NEON(const float *lhs, const float *rhs, size_t size); float MinusInnerProductFp32NEON(const float *lhs, const float *rhs, size_t size); @@ -49,7 +49,7 @@ float MinusInnerProductFp32Scalar(const float *lhs, const float *rhs, //! Compute the distance between matrix and query (FP32, M=1, N=1) void InnerProductMatrix::Compute(const float *m, const float *q, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) *out = InnerProductFp32NEON(m, q, dim); #else #if defined(__AVX512F__) @@ -80,7 +80,7 @@ void InnerProductMatrix::Compute(const float *m, const float *q, void MinusInnerProductMatrix::Compute(const float *m, const float *q, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) *out = MinusInnerProductFp32NEON(m, q, dim); #else #if defined(__AVX512F__) diff --git a/src/ailego/math/inner_product_matrix_fp32_neon.cc b/src/ailego/math/inner_product_matrix_fp32_neon.cc index c457b3ea2..e21fd3abf 100644 --- a/src/ailego/math/inner_product_matrix_fp32_neon.cc +++ b/src/ailego/math/inner_product_matrix_fp32_neon.cc @@ -22,7 +22,7 @@ namespace ailego { //-------------------------------------------------- // Dense //-------------------------------------------------- -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) float InnerProductFp32NEON(const float *lhs, const float *rhs, size_t size) { const float *last = lhs + size; const float *last_aligned = lhs + ((size >> 3) << 3); diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc index f48626a3f..37c8a1daf 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc @@ -18,7 +18,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, size_t size, float *sql, float *sqr); #endif @@ -98,7 +98,7 @@ void MipsSquaredEuclideanDistanceMatrix::Compute( void MipsSquaredEuclideanDistanceMatrix::Compute( const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, float *out) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) float u2{0.0f}; float v2{0.0f}; float sum = InnerProductAndSquaredNormFp32NEON(p, q, dim, &u2, &v2); diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc index 6491f2260..e5bff681e 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) //! Compute the Inner Product between p and q, and each Squared L2-Norm value float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, size_t size, float *sql, float *sqr) { diff --git a/src/ailego/math/norm1_matrix.h b/src/ailego/math/norm1_matrix.h index 7e8d9cbc8..f8d634e32 100644 --- a/src/ailego/math/norm1_matrix.h +++ b/src/ailego/math/norm1_matrix.h @@ -116,7 +116,8 @@ struct Norm1Matrix< } }; -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) /*! L1-Norm Matrix (FP32, M=1) */ template <> @@ -129,6 +130,8 @@ struct Norm1Matrix { }; #endif // __SSE__ || (__ARM_NEON && __aarch64__) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON +// specialization to gcc/clang aarch64. #if (defined(__F16C__) && defined(__AVX__)) || \ (defined(__ARM_NEON) && defined(__aarch64__)) /*! L1-Norm Matrix (FP16, M=1) diff --git a/src/ailego/math/norm1_matrix_fp16.cc b/src/ailego/math/norm1_matrix_fp16.cc index e75b3e0a8..0c092bbb7 100644 --- a/src/ailego/math/norm1_matrix_fp16.cc +++ b/src/ailego/math/norm1_matrix_fp16.cc @@ -67,6 +67,9 @@ static const __m512 ABS_MASK_FP32_AVX512 = //! Calculate sum of absolute (NEON) #define SA_FP16_NEON(v_m, v_sum) v_sum = vaddq_f16(vabsq_f16(v_m), v_sum); +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the NEON FP16 +// kernel is gated to gcc/clang aarch64. The generic Float16 path is used +// on MSVC ARM64. #if (defined(__F16C__) && defined(__AVX__)) || \ (defined(__ARM_NEON) && defined(__aarch64__)) //! Compute the L1-norm of vectors (FP16, M=1) diff --git a/src/ailego/math/norm1_matrix_fp32.cc b/src/ailego/math/norm1_matrix_fp32.cc index 2e7279118..b4194b2f7 100644 --- a/src/ailego/math/norm1_matrix_fp32.cc +++ b/src/ailego/math/norm1_matrix_fp32.cc @@ -56,11 +56,12 @@ namespace ailego { //! Calculate sum of absolute (NEON) #define SA_FP32_NEON(v_m, v_sum) v_sum = vaddq_f32(vabsq_f32(v_m), v_sum); -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) //! Compute the L1-norm of vectors (FP32, M=1) void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) NORM_FP32_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) diff --git a/src/ailego/math/norm2_matrix.h b/src/ailego/math/norm2_matrix.h index 3c905147d..415f39551 100644 --- a/src/ailego/math/norm2_matrix.h +++ b/src/ailego/math/norm2_matrix.h @@ -371,7 +371,8 @@ struct SquaredNorm2Matrix= 2>::type> { } }; -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) /*! L2-Norm Matrix (FP32, M=1) */ template <> @@ -395,6 +396,8 @@ struct SquaredNorm2Matrix { }; #endif // __SSE__ || (__ARM_NEON && __aarch64__) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON +// specialization to gcc/clang aarch64. #if (defined(__F16C__) && defined(__AVX__)) || \ (defined(__ARM_NEON) && defined(__aarch64__)) /*! L2-Norm Matrix (FP16, M=1) diff --git a/src/ailego/math/norm2_matrix_fp16.cc b/src/ailego/math/norm2_matrix_fp16.cc index 6bb8dd06c..a87437d76 100644 --- a/src/ailego/math/norm2_matrix_fp16.cc +++ b/src/ailego/math/norm2_matrix_fp16.cc @@ -52,6 +52,8 @@ namespace ailego { //! Calculate sum of squared (NEON) #define SS_FP16_NEON(v_m, v_sum) v_sum = vfmaq_f16(v_sum, v_m, v_m); +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the NEON FP16 +// kernel is gated to gcc/clang aarch64. #if (defined(__F16C__) && defined(__AVX__)) || \ (defined(__ARM_NEON) && defined(__aarch64__)) //! Compute the L2-norm of vectors (FP16, M=1) diff --git a/src/ailego/math/norm2_matrix_fp32.cc b/src/ailego/math/norm2_matrix_fp32.cc index 8cc76c1f5..b1c1ad399 100644 --- a/src/ailego/math/norm2_matrix_fp32.cc +++ b/src/ailego/math/norm2_matrix_fp32.cc @@ -43,11 +43,12 @@ namespace ailego { //! Calculate sum of squared (NEON) #define SS_FP32_NEON(v_m, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_m); -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) //! Compute the L2-norm of vectors (FP32, M=1) void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) NORM_FP32_1_NEON(m, dim, out, std::sqrt) #else #if defined(__AVX512F__) @@ -69,7 +70,7 @@ void Norm2Matrix::Compute(const ValueType *m, size_t dim, //! Compute the squared L2-norm of vectors (FP32, M=1) void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) NORM_FP32_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) diff --git a/src/ailego/math/normalizer.cc b/src/ailego/math/normalizer.cc index a31a9f350..a25b75454 100644 --- a/src/ailego/math/normalizer.cc +++ b/src/ailego/math/normalizer.cc @@ -17,7 +17,8 @@ namespace zvec { namespace ailego { -#if (defined(__ARM_NEON) && defined(__aarch64__)) +#if ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) static inline void NormalizeNEON(float *arr, size_t dim, float norm) { float *last = arr + dim; float *last_aligned = arr + ((dim >> 3) << 3); @@ -43,6 +44,11 @@ static inline void NormalizeNEON(float *arr, size_t dim, float norm) { } } +// MSVC ARM64 does not declare `float16_t` unless ARMv8.2 FP16 is enabled +// (`_M_ARM64FP16` / `/arch:armv8.2`). Skip the FP16 NEON variants on MSVC +// ARM64 — the FP16 `Normalizer::Compute` dispatch below falls +// through to the scalar conversion path. +#if !defined(_MSC_VER) #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) static inline void NormalizeNEON(float16_t *arr, size_t dim, float norm) { float16_t *last = arr + dim; @@ -114,6 +120,7 @@ static inline void NormalizeNEON(float16_t *arr, size_t dim, float norm) { } } #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +#endif // !_MSC_VER (FP16 NEON: gcc/clang aarch64 only) #endif // __ARM_NEON && __aarch64__ #if defined(__AVX__) @@ -392,10 +399,11 @@ static inline void NormalizeSSE(float *arr, size_t dim, float norm) { } #endif // __SSE__ -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) //! Compute the norm of vector void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) NormalizeNEON(arr, dim, norm); #else #if defined(__AVX512F__) @@ -415,6 +423,10 @@ void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { } #endif // __SSE__ || (__ARM_NEON && __aarch64__) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 +// `NormalizeNEON` variants are not defined there (see top of file). +// Exclude MSVC from this specialization so MSVC ARM64 falls back to the +// generic uint16_t-based Float16 normalize path. #if (defined(__F16C__) && defined(__AVX__)) || \ (defined(__ARM_NEON) && defined(__aarch64__)) //! Compute the norm of vector diff --git a/src/ailego/math/normalizer.h b/src/ailego/math/normalizer.h index 2c191b0e7..fd756d9d5 100644 --- a/src/ailego/math/normalizer.h +++ b/src/ailego/math/normalizer.h @@ -51,7 +51,8 @@ struct Normalizer { } }; -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) /*! Normalizer (FP32) */ template <> @@ -80,6 +81,8 @@ struct Normalizer { }; #endif // __SSE__ || (__ARM_NEON && __aarch64__) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 NEON +// `Normalizer` specialization is gated to gcc/clang aarch64. #if (defined(__F16C__) && defined(__AVX__)) || \ (defined(__ARM_NEON) && defined(__aarch64__)) /*! Normalizer (FP16) diff --git a/src/ailego/utility/bitset_helper.cc b/src/ailego/utility/bitset_helper.cc index 19be34847..47d5c6238 100644 --- a/src/ailego/utility/bitset_helper.cc +++ b/src/ailego/utility/bitset_helper.cc @@ -23,7 +23,7 @@ #define bitset_popcount64 _mm_popcnt_u64 #endif // !__SSE4_2__ -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) static inline void bitset_and(uint32_t *lhs, const uint32_t *rhs, size_t size) { uint32_t *last = lhs + size; uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1480,7 +1480,8 @@ static inline bool bitset_test_none(const uint32_t *lhs, size_t size) { #endif // AILEGO_M64 #endif // __AVX2__ -#if (defined(__ARM_NEON) && defined(__aarch64__)) +#if ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ + (defined(__aarch64__) || defined(_M_ARM64))) static inline size_t bitset_cardinality(const uint32_t *lhs, size_t size) { const uint32_t *last = lhs + size; const uint32_t *last_aligned = lhs + ((size >> 2) << 2); diff --git a/src/ailego/utility/float_helper.cc b/src/ailego/utility/float_helper.cc index bc07eec25..90ecb92e8 100644 --- a/src/ailego/utility/float_helper.cc +++ b/src/ailego/utility/float_helper.cc @@ -22,7 +22,10 @@ // #define float32(x) _cvtsh_ss(x) // #endif // __F16C__ && __AVX__ -#if defined(__aarch64__) +// MSVC ARM64 lacks the GCC/Clang `__fp16` extension type, so keep this +// path gated on `__aarch64__` (predefined only by GCC/Clang on AArch64). +// MSVC ARM64 falls through to the F16C/scalar paths below. +#if defined(__aarch64__) && !defined(_MSC_VER) static inline float float32(uint16_t val) { __fp16 f; memcpy(&f, &val, sizeof(val)); diff --git a/src/ailego/version.i b/src/ailego/version.i index c1b14be2e..f8e78d443 100644 --- a/src/ailego/version.i +++ b/src/ailego/version.i @@ -225,7 +225,7 @@ #define AILEGO_VERSION_OPENMP "" #endif -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) #define AILEGO_VERSION_SIMD " Arm Neon Instruction Set\n" #elif defined(__AVX512FP16__) #define AILEGO_VERSION_SIMD " AVX-512FP16 Instruction Set\n" diff --git a/src/include/zvec/ailego/buffer/concurrentqueue.h b/src/include/zvec/ailego/buffer/concurrentqueue.h index f7f3d77ed..1bff8b08e 100644 --- a/src/include/zvec/ailego/buffer/concurrentqueue.h +++ b/src/include/zvec/ailego/buffer/concurrentqueue.h @@ -129,8 +129,9 @@ static inline thread_id_t thread_id() { } } // namespace details } // namespace moodycamel -#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || \ - (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__) || \ +#elif defined(__arm__) || defined(_M_ARM) || \ + (defined(__aarch64__) || defined(_M_ARM64)) || \ + (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__) || \ defined(MOODYCAMEL_NO_THREAD_LOCAL) namespace moodycamel { namespace details { @@ -293,13 +294,14 @@ inline thread_id_t thread_id() { // support thread_local either. Finally, iOS/ARM doesn't have support for it // either, and g++/ARM allows it to compile but it's unconfirmed to actually // work -#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && \ - (!defined(__MINGW32__) && !defined(__MINGW64__) || \ - !defined(__WINPTHREADS_VERSION)) && \ - (!defined(__GNUC__) || __GNUC__ > 4 || \ - (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && \ - (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && \ - !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__) +#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && \ + (!defined(__MINGW32__) && !defined(__MINGW64__) || \ + !defined(__WINPTHREADS_VERSION)) && \ + (!defined(__GNUC__) || __GNUC__ > 4 || \ + (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && \ + (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && \ + !defined(_M_ARM) && !(defined(__aarch64__) || defined(_M_ARM64)) && \ + !defined(__MVS__) // Assume `thread_local` is fully supported in all other C++11 // compilers/platforms #define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // tentatively enabled for now; diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 2b1c1a0e1..0112f5979 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -30,13 +30,16 @@ #if defined(_MSC_VER) #include +#if defined(_M_ARM64) +#include +#endif #else #include #include #if defined(__x86_64__) || defined(__i386) #include #endif -#if defined(__ARM_NEON) +#if (defined(__ARM_NEON) || defined(_M_ARM64)) #include #endif #if defined(__ARM_FEATURE_CRC32) @@ -111,7 +114,8 @@ extern "C" { #endif #if defined(__GNUC__) -#if defined(__x86_64__) || defined(__aarch64__) || defined(__ppc64__) +#if defined(__x86_64__) || (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ppc64__) #define AILEGO_M64 #else #define AILEGO_M32 @@ -236,10 +240,12 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_popcount ailego_popcount32 #endif // AILEGO_M64 -#if defined(__arm__) || defined(__aarch64__) +#if defined(__arm__) || (defined(__aarch64__) || defined(_M_ARM64)) // ARMv7 Architecture Reference Manual (for YIELD) // ARM Compiler toolchain Compiler Reference (for __yield() instrinsic) -#if defined(__CC_ARM) +#if defined(__CC_ARM) || defined(_MSC_VER) +// ARM Compiler toolchain and MSVC both expose the intrinsic `__yield()` +// for the AArch64/ARMv7 YIELD instruction. #define ailego_yield() __yield() #else #define ailego_yield() __asm__ __volatile__("yield") @@ -281,11 +287,12 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 32) #elif defined(__SSE__) #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 16) -#elif defined(__ARM_NEON) +#elif (defined(__ARM_NEON) || defined(_M_ARM64)) #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 16) #endif #endif // !ailego_malloc -#if (defined(__SSE__) || defined(__ARM_NEON)) && !defined(ailego_free) +#if (defined(__SSE__) || (defined(__ARM_NEON) || defined(_M_ARM64))) && \ + !defined(ailego_free) #define ailego_free ailego_aligned_free #endif #endif // !__SANITIZE_ADDRESS__ diff --git a/src/include/zvec/ailego/utility/float_helper.h b/src/include/zvec/ailego/utility/float_helper.h index 5dc2fe69a..d32eb3404 100644 --- a/src/include/zvec/ailego/utility/float_helper.h +++ b/src/include/zvec/ailego/utility/float_helper.h @@ -52,7 +52,10 @@ struct FloatHelper { } }; -#if !defined(__aarch64__) +// The `#else` branch below stores `Float16::value_` as `__fp16` — a GCC/Clang +// extension type that MSVC does not provide even on ARM64. Keep the uint16_t +// storage path for MSVC (including MSVC ARM64) so the wrapper compiles. +#if !defined(__aarch64__) || defined(_MSC_VER) /*! Half-Precision Floating Point */ class Float16 { diff --git a/thirdparty/FastPFOR/CMakeLists.txt b/thirdparty/FastPFOR/CMakeLists.txt index 6ff7b8020..4fb95f2f2 100644 --- a/thirdparty/FastPFOR/CMakeLists.txt +++ b/thirdparty/FastPFOR/CMakeLists.txt @@ -52,3 +52,12 @@ cc_library( DEFS ${FASTPFOR_EXTRA_DEFS} CXXFLAGS ${FASTPFOR_EXTRA_CXXFLAGS} ) + +# Mark SIMDe's x86 emulation headers as a SYSTEM include so consumers of +# FastPFOR (e.g. src/db's bitpacked posting-list translation units) do not +# inherit SIMDe's warnings under strict /W4 /WX builds. Without this MSVC +# ARM64 fails on C4189 / C4127 / C4324 from simde/x86/mmx.h. +if(_FASTPFOR_NEEDS_SIMDE) + set_property(TARGET FastPFOR APPEND PROPERTY + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${simde_SOURCE_DIR}") +endif() From 333ac6e062ac4afd3cd89e3be55e1d748907eff1 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Tue, 21 Jul 2026 21:45:01 +0200 Subject: [PATCH 4/9] refactor(ailego): use AILEGO_HAVE_NEON/AILEGO_ARM64 feature macros Addresses review feedback that the inline _M_ARM64 preprocessor guards were confusing and needed simplifying. The NEON-enablement change had mechanically expanded `defined(__ARM_NEON)` -> `(defined(__ARM_NEON) || defined(_M_ARM64))` and `defined(__aarch64__)` -> `(defined(__aarch64__) || defined(_M_ARM64))` at every site, producing long, hard-to-read compound expressions -- one of which (`(NEON||M) && !(aarch64||M)`) was even logically dead on MSVC ARM64. Introduce two self-documenting feature macros in platform.h, defined once: AILEGO_ARM64 - 64-bit ARM (AArch64): __aarch64__ (GCC/Clang) or _M_ARM64 (MSVC) AILEGO_HAVE_NEON - NEON intrinsics available: __ARM_NEON (GCC/Clang) or _M_ARM64 (MSVC ARM64, ARMv8 baseline) and replace the verbose guards across the ailego math kernels, platform.h, cpu_features.cc and bitset_helper.cc with them. This is a pure readability change -- every guard expands to the same truth value on every target as before, so behaviour is unchanged. cpu_features.cc now includes platform.h so it can use the macros. Verified on Windows ARM64 (MSVC): zvec_ailego compiles, and a preprocessor probe confirms both macros resolve true so the NEON kernels are still compiled in (not the scalar fallback). --- src/ailego/CMakeLists.txt | 6 ++-- src/ailego/internal/cpu_features.cc | 9 +++--- src/ailego/math/distance_matrix_accum_fp32.i | 4 +-- src/ailego/math/distance_matrix_fp32.i | 2 +- ...euclidean_distance_matrix_fp32_dispatch.cc | 4 +-- .../euclidean_distance_matrix_fp32_neon.cc | 2 +- .../inner_product_matrix_fp32_dispatch.cc | 6 ++-- .../math/inner_product_matrix_fp32_neon.cc | 2 +- ...euclidean_distance_matrix_fp32_dispatch.cc | 4 +-- ...ips_euclidean_distance_matrix_fp32_neon.cc | 2 +- src/ailego/math/norm1_matrix.h | 3 +- src/ailego/math/norm1_matrix_fp32.cc | 5 ++-- src/ailego/math/norm2_matrix.h | 3 +- src/ailego/math/norm2_matrix_fp32.cc | 7 ++--- src/ailego/math/normalizer.cc | 8 ++---- src/ailego/math/normalizer.h | 3 +- src/ailego/utility/bitset_helper.cc | 5 ++-- src/ailego/version.i | 2 +- src/include/zvec/ailego/internal/platform.h | 28 ++++++++++++++----- 19 files changed, 55 insertions(+), 50 deletions(-) diff --git a/src/ailego/CMakeLists.txt b/src/ailego/CMakeLists.txt index 606b0b5e9..cbc0be98e 100644 --- a/src/ailego/CMakeLists.txt +++ b/src/ailego/CMakeLists.txt @@ -116,9 +116,9 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) # MSVC on ARM64: NEON is the ARMv8 baseline and is always enabled, # so no `-march` flag is required (MSVC does not accept GCC-style # `-march=` anyway). The NEON math kernels still get compiled via - # the ALL_SRCS glob above; their `#if defined(__ARM_NEON)` guards - # were extended in this PR to also accept `_M_ARM64` so the bodies - # actually emit code under MSVC. + # the ALL_SRCS glob above; their guards use the AILEGO_HAVE_NEON / + # AILEGO_ARM64 feature macros (defined in platform.h), which resolve + # to true on MSVC ARM64 so the bodies actually emit code. endif() endif() endif() diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 395e6fc13..9848b5527 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -14,11 +14,11 @@ #include "cpu_features.h" #include +#include #if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) #include -#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && \ - !(defined(__aarch64__) || defined(_M_ARM64)) +#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(AILEGO_ARM64) #include #endif @@ -49,8 +49,7 @@ CpuFeatures::CpuFlags::CpuFlags(void) L7_ECX = l7[2]; L7_EDX = l7[3]; } -#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && \ - !(defined(__aarch64__) || defined(_M_ARM64)) +#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(AILEGO_ARM64) CpuFeatures::CpuFlags::CpuFlags(void) : L1_ECX(0), L1_EDX(0), L7_EBX(0), L7_ECX(0), L7_EDX(0) { uint32_t eax, ebx, ecx, edx; @@ -338,7 +337,7 @@ bool CpuFeatures::HYPERVISOR(void) { const char *CpuFeatures::Intrinsics(void) { return "" -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) "Neon" #if defined(__ARM_FEATURE_CRC32) "+CRC" diff --git a/src/ailego/math/distance_matrix_accum_fp32.i b/src/ailego/math/distance_matrix_accum_fp32.i index 913784076..7ae6bce2c 100644 --- a/src/ailego/math/distance_matrix_accum_fp32.i +++ b/src/ailego/math/distance_matrix_accum_fp32.i @@ -30,7 +30,7 @@ _mm512_castps_si512(b))) #endif // __AVX512DQ__ -#if (defined(__ARM_NEON) || defined(_M_ARM64)) && !(defined(__aarch64__) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) && !defined(AILEGO_ARM64) static inline float32_t vaddvq_f32(float32x4_t v) { float32x2_t s = vadd_f32(vget_low_f32(v), vget_high_f32(v)); return vget_lane_f32(vpadd_f32(s, s), 0); @@ -42,7 +42,7 @@ static inline int32_t vaddvq_s32(int32x4_t v) { } #endif //__ARM_NEON && !__aarch64__ -#if (defined(__aarch64__) || defined(_M_ARM64)) +#if defined(AILEGO_ARM64) #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A64 #else #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A32 diff --git a/src/ailego/math/distance_matrix_fp32.i b/src/ailego/math/distance_matrix_fp32.i index f4e6a16ab..10cac5e0c 100644 --- a/src/ailego/math/distance_matrix_fp32.i +++ b/src/ailego/math/distance_matrix_fp32.i @@ -26,7 +26,7 @@ _mm256_insertf128_ps(_mm256_castps128_ps256(b), (a), 1) #endif // __AVX__ -#if (defined(__ARM_NEON) || defined(_M_ARM64)) && !(defined(__aarch64__) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) && !defined(AILEGO_ARM64) #define vdupq_laneq_f32(a, b) vdupq_n_f32(vgetq_lane_f32(a, b)) #endif // __ARM_NEON && __aarch64__ diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc index f0650a08e..9dabbe757 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc @@ -18,7 +18,7 @@ namespace zvec { namespace ailego { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, size_t size, float *out); #endif @@ -49,7 +49,7 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) SquaredEuclideanDistanceFp32NEON(m, q, dim, out); #else #if defined(__AVX512F__) diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc index 14ce90767..8088bb7e1 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) //! Squared Euclidean Distance void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, size_t size, float *out) { diff --git a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc index 32540296b..867b9a3e8 100644 --- a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc +++ b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc @@ -20,7 +20,7 @@ namespace ailego { //-------------------------------------------------- // Dense //-------------------------------------------------- -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) float InnerProductFp32NEON(const float *lhs, const float *rhs, size_t size); float MinusInnerProductFp32NEON(const float *lhs, const float *rhs, size_t size); @@ -49,7 +49,7 @@ float MinusInnerProductFp32Scalar(const float *lhs, const float *rhs, //! Compute the distance between matrix and query (FP32, M=1, N=1) void InnerProductMatrix::Compute(const float *m, const float *q, size_t dim, float *out) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) *out = InnerProductFp32NEON(m, q, dim); #else #if defined(__AVX512F__) @@ -80,7 +80,7 @@ void InnerProductMatrix::Compute(const float *m, const float *q, void MinusInnerProductMatrix::Compute(const float *m, const float *q, size_t dim, float *out) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) *out = MinusInnerProductFp32NEON(m, q, dim); #else #if defined(__AVX512F__) diff --git a/src/ailego/math/inner_product_matrix_fp32_neon.cc b/src/ailego/math/inner_product_matrix_fp32_neon.cc index e21fd3abf..4f8028359 100644 --- a/src/ailego/math/inner_product_matrix_fp32_neon.cc +++ b/src/ailego/math/inner_product_matrix_fp32_neon.cc @@ -22,7 +22,7 @@ namespace ailego { //-------------------------------------------------- // Dense //-------------------------------------------------- -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) float InnerProductFp32NEON(const float *lhs, const float *rhs, size_t size) { const float *last = lhs + size; const float *last_aligned = lhs + ((size >> 3) << 3); diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc index 37c8a1daf..c8f50fb8c 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc @@ -18,7 +18,7 @@ namespace zvec { namespace ailego { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, size_t size, float *sql, float *sqr); #endif @@ -98,7 +98,7 @@ void MipsSquaredEuclideanDistanceMatrix::Compute( void MipsSquaredEuclideanDistanceMatrix::Compute( const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, float *out) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) float u2{0.0f}; float v2{0.0f}; float sum = InnerProductAndSquaredNormFp32NEON(p, q, dim, &u2, &v2); diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc index e5bff681e..78198ecf1 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) //! Compute the Inner Product between p and q, and each Squared L2-Norm value float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, size_t size, float *sql, float *sqr) { diff --git a/src/ailego/math/norm1_matrix.h b/src/ailego/math/norm1_matrix.h index f8d634e32..3223e2fca 100644 --- a/src/ailego/math/norm1_matrix.h +++ b/src/ailego/math/norm1_matrix.h @@ -116,8 +116,7 @@ struct Norm1Matrix< } }; -#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) /*! L1-Norm Matrix (FP32, M=1) */ template <> diff --git a/src/ailego/math/norm1_matrix_fp32.cc b/src/ailego/math/norm1_matrix_fp32.cc index b4194b2f7..78b24b0f3 100644 --- a/src/ailego/math/norm1_matrix_fp32.cc +++ b/src/ailego/math/norm1_matrix_fp32.cc @@ -56,12 +56,11 @@ namespace ailego { //! Calculate sum of absolute (NEON) #define SA_FP32_NEON(v_m, v_sum) v_sum = vaddq_f32(vabsq_f32(v_m), v_sum); -#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) //! Compute the L1-norm of vectors (FP32, M=1) void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) NORM_FP32_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) diff --git a/src/ailego/math/norm2_matrix.h b/src/ailego/math/norm2_matrix.h index 415f39551..568fe4151 100644 --- a/src/ailego/math/norm2_matrix.h +++ b/src/ailego/math/norm2_matrix.h @@ -371,8 +371,7 @@ struct SquaredNorm2Matrix= 2>::type> { } }; -#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) /*! L2-Norm Matrix (FP32, M=1) */ template <> diff --git a/src/ailego/math/norm2_matrix_fp32.cc b/src/ailego/math/norm2_matrix_fp32.cc index b1c1ad399..2fa59618a 100644 --- a/src/ailego/math/norm2_matrix_fp32.cc +++ b/src/ailego/math/norm2_matrix_fp32.cc @@ -43,12 +43,11 @@ namespace ailego { //! Calculate sum of squared (NEON) #define SS_FP32_NEON(v_m, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_m); -#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) //! Compute the L2-norm of vectors (FP32, M=1) void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) NORM_FP32_1_NEON(m, dim, out, std::sqrt) #else #if defined(__AVX512F__) @@ -70,7 +69,7 @@ void Norm2Matrix::Compute(const ValueType *m, size_t dim, //! Compute the squared L2-norm of vectors (FP32, M=1) void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) NORM_FP32_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) diff --git a/src/ailego/math/normalizer.cc b/src/ailego/math/normalizer.cc index a25b75454..1a65c41ca 100644 --- a/src/ailego/math/normalizer.cc +++ b/src/ailego/math/normalizer.cc @@ -17,8 +17,7 @@ namespace zvec { namespace ailego { -#if ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) static inline void NormalizeNEON(float *arr, size_t dim, float norm) { float *last = arr + dim; float *last_aligned = arr + ((dim >> 3) << 3); @@ -399,11 +398,10 @@ static inline void NormalizeSSE(float *arr, size_t dim, float norm) { } #endif // __SSE__ -#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) //! Compute the norm of vector void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) NormalizeNEON(arr, dim, norm); #else #if defined(__AVX512F__) diff --git a/src/ailego/math/normalizer.h b/src/ailego/math/normalizer.h index fd756d9d5..16d9f0050 100644 --- a/src/ailego/math/normalizer.h +++ b/src/ailego/math/normalizer.h @@ -51,8 +51,7 @@ struct Normalizer { } }; -#if defined(__SSE__) || ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) /*! Normalizer (FP32) */ template <> diff --git a/src/ailego/utility/bitset_helper.cc b/src/ailego/utility/bitset_helper.cc index 47d5c6238..196678fde 100644 --- a/src/ailego/utility/bitset_helper.cc +++ b/src/ailego/utility/bitset_helper.cc @@ -23,7 +23,7 @@ #define bitset_popcount64 _mm_popcnt_u64 #endif // !__SSE4_2__ -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) static inline void bitset_and(uint32_t *lhs, const uint32_t *rhs, size_t size) { uint32_t *last = lhs + size; uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1480,8 +1480,7 @@ static inline bool bitset_test_none(const uint32_t *lhs, size_t size) { #endif // AILEGO_M64 #endif // __AVX2__ -#if ((defined(__ARM_NEON) || defined(_M_ARM64)) && \ - (defined(__aarch64__) || defined(_M_ARM64))) +#if (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) static inline size_t bitset_cardinality(const uint32_t *lhs, size_t size) { const uint32_t *last = lhs + size; const uint32_t *last_aligned = lhs + ((size >> 2) << 2); diff --git a/src/ailego/version.i b/src/ailego/version.i index f8e78d443..d6347cc3f 100644 --- a/src/ailego/version.i +++ b/src/ailego/version.i @@ -225,7 +225,7 @@ #define AILEGO_VERSION_OPENMP "" #endif -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) #define AILEGO_VERSION_SIMD " Arm Neon Instruction Set\n" #elif defined(__AVX512FP16__) #define AILEGO_VERSION_SIMD " AVX-512FP16 Instruction Set\n" diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 0112f5979..fac32fed4 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -14,6 +14,22 @@ #pragma once +// Architecture / SIMD feature detection, defined once so the guards below +// (and across the codebase) read clearly instead of repeating long +// compiler-macro disjunctions: +// +// AILEGO_ARM64 - 64-bit ARM (AArch64). GCC/Clang predefine __aarch64__; +// MSVC predefines _M_ARM64. +// AILEGO_HAVE_NEON - NEON intrinsics available. GCC/Clang predefine +// __ARM_NEON; MSVC ARM64 has NEON (ARMv8 baseline) and +// predefines only _M_ARM64. +#if defined(__aarch64__) || defined(_M_ARM64) +#define AILEGO_ARM64 1 +#endif +#if defined(__ARM_NEON) || defined(_M_ARM64) +#define AILEGO_HAVE_NEON 1 +#endif + #if defined(_WIN32) || defined(_WIN64) #include #endif @@ -39,7 +55,7 @@ #if defined(__x86_64__) || defined(__i386) #include #endif -#if (defined(__ARM_NEON) || defined(_M_ARM64)) +#if defined(AILEGO_HAVE_NEON) #include #endif #if defined(__ARM_FEATURE_CRC32) @@ -114,8 +130,7 @@ extern "C" { #endif #if defined(__GNUC__) -#if defined(__x86_64__) || (defined(__aarch64__) || defined(_M_ARM64)) || \ - defined(__ppc64__) +#if defined(__x86_64__) || defined(AILEGO_ARM64) || defined(__ppc64__) #define AILEGO_M64 #else #define AILEGO_M32 @@ -240,7 +255,7 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_popcount ailego_popcount32 #endif // AILEGO_M64 -#if defined(__arm__) || (defined(__aarch64__) || defined(_M_ARM64)) +#if defined(__arm__) || defined(AILEGO_ARM64) // ARMv7 Architecture Reference Manual (for YIELD) // ARM Compiler toolchain Compiler Reference (for __yield() instrinsic) #if defined(__CC_ARM) || defined(_MSC_VER) @@ -287,12 +302,11 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 32) #elif defined(__SSE__) #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 16) -#elif (defined(__ARM_NEON) || defined(_M_ARM64)) +#elif defined(AILEGO_HAVE_NEON) #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 16) #endif #endif // !ailego_malloc -#if (defined(__SSE__) || (defined(__ARM_NEON) || defined(_M_ARM64))) && \ - !defined(ailego_free) +#if (defined(__SSE__) || defined(AILEGO_HAVE_NEON)) && !defined(ailego_free) #define ailego_free ailego_aligned_free #endif #endif // !__SANITIZE_ADDRESS__ From 245421a9e8ffcff8d077d35697db5c2711bc15e4 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Tue, 21 Jul 2026 22:46:04 +0200 Subject: [PATCH 5/9] fix(ailego): keep cpuid path gated to x86, not just non-ARM cpu_features.cc selected the `__get_cpuid` / `__cpuid_count` constructor (and included ) under `!defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(AILEGO_ARM64)`. That denylist is true on non-x86, non-ARM targets such as riscv64 and ppc64, where does not exist and the __get_cpuid intrinsics are unavailable -- a compile break (the linux-riscv CI job builds this file). Restore the original x86 allowlist `defined(__x86_64__) || defined(__i386__)` for both the include and the constructor. This still lands MSVC ARM64 on the empty-stub constructor (it defines neither __x86_64__ nor __i386__), which was the intent, while non-x86/non-ARM arches correctly fall through to the stub again. Reported by the Copilot PR reviewer. --- src/ailego/internal/cpu_features.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 9848b5527..c8ef8fb97 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -18,7 +18,7 @@ #if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) #include -#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(AILEGO_ARM64) +#elif defined(__x86_64__) || defined(__i386__) #include #endif @@ -49,7 +49,7 @@ CpuFeatures::CpuFlags::CpuFlags(void) L7_ECX = l7[2]; L7_EDX = l7[3]; } -#elif !defined(_MSC_VER) && !defined(__ARM_ARCH) && !defined(AILEGO_ARM64) +#elif defined(__x86_64__) || defined(__i386__) CpuFeatures::CpuFlags::CpuFlags(void) : L1_ECX(0), L1_EDX(0), L7_EBX(0), L7_ECX(0), L7_EDX(0) { uint32_t eax, ebx, ecx, edx; From 92c690fb59b18cda3f2786cd112cba93e71c73a8 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Thu, 13 Aug 2026 21:03:56 +0200 Subject: [PATCH 6/9] fix(turbo): stop NEON FHT rotator emitting unwritten output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FHT rotator merged from main guards every `zvec::turbo::neon` function body with `__ARM_NEON && __aarch64__` and falls back to a no-op `(void)` stub. MSVC ARM64 defines neither macro, so `fht_rotate_neon` / `fht_unrotate_neon` returned without ever writing `out`, leaving the caller's buffer uninitialized rather than rotated. This was masked only by accident: `CpuFeatures::NEON()` also tested `__ARM_NEON`, so it reported false on MSVC ARM64 and turbo.cc fell back to the scalar rotator. That made `NEON()` disagree with `Intrinsics()` and `version.i`, which already report "Neon" from `AILEGO_HAVE_NEON` — so the obvious follow-up fix to `NEON()` would have activated the no-op path and silently corrupted results. Fix both halves together: - Guard the turbo NEON kernels with `AILEGO_HAVE_NEON && AILEGO_ARM64` so the real intrinsics compile on MSVC ARM64, and make every `#else` branch delegate to the scalar implementation instead of doing nothing. - Make `CpuFeatures::NEON()` use `AILEGO_HAVE_NEON`, consistent with `Intrinsics()` / `version.i`. - Build the sign-flip lane mask with `vld1q_u32`; MSVC models `uint32x4_t` as a union, so GCC-style brace initialization does not compile there. This surfaced once the kernels were actually built for MSVC ARM64. Also apply the same macros to the ailego FHT kernels (`fht_neon.cc`, `fht_dispatch.cc`), which were likewise left on the raw macros by the merge and therefore fell back to scalar on MSVC ARM64, and fix `mips_euclidean_distance_matrix_fp32_dispatch.cc`, where one call site still tested `__ARM_NEON` while its own forward declaration and sibling function already used `AILEGO_HAVE_NEON`. Verified on Windows ARM64 (MSVC): the NEON kernels and rotator now match the scalar reference bit-for-bit across power-of-2, non-power-of-2 and tail-remainder dimensions, and rotate/unrotate round-trips recover the input. Against the previous code the same check reports all-zero output, confirming the no-op. Changed files also compile clean for x64. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ailego/internal/cpu_features.cc | 6 ++- src/ailego/math/fht_dispatch.cc | 15 +++--- src/ailego/math/fht_neon.cc | 11 ++-- ...euclidean_distance_matrix_fp32_dispatch.cc | 6 +-- src/turbo/distance/neon/rotate/fht/fht.cc | 54 +++++++++---------- 5 files changed, 50 insertions(+), 42 deletions(-) diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 65b82b838..2173a43d7 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -346,7 +346,11 @@ bool CpuFeatures::HYPERVISOR(void) { bool CpuFeatures::NEON(void) { #if defined(__aarch64__) && defined(__linux__) return !!(getauxval(AT_HWCAP) & HWCAP_ASIMD); -#elif defined(__ARM_NEON) +#elif defined(AILEGO_HAVE_NEON) + // NEON is part of the ARMv8 baseline, so it is unconditionally present when + // the target has it (including MSVC ARM64, which predefines only _M_ARM64). + // This must agree with Intrinsics()/version.i, which report "Neon" from the + // same macro. return true; #else return false; diff --git a/src/ailego/math/fht_dispatch.cc b/src/ailego/math/fht_dispatch.cc index b1e9c4aac..1c1cea834 100644 --- a/src/ailego/math/fht_dispatch.cc +++ b/src/ailego/math/fht_dispatch.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include "fht.h" namespace zvec { @@ -41,7 +42,7 @@ void fht_kacs_walk_avx512(float *data, size_t len); void fht_inv_kacs_walk_avx512(float *data, size_t len); void fht_inplace_avx512(float *data, size_t n); #endif -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim); void fht_kacs_walk_neon(float *data, size_t len); void fht_inv_kacs_walk_neon(float *data, size_t len); @@ -52,7 +53,7 @@ void fht_inv_kacs_walk_neon(float *data, size_t len); // ============================================================================ void fht_flip_sign(const uint8_t *flip, float *data, size_t dim) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) fht_flip_sign_neon(flip, data, dim); #else #if defined(__AVX512F__) @@ -75,11 +76,11 @@ void fht_flip_sign(const uint8_t *flip, float *data, size_t dim) { } #endif fht_flip_sign_scalar(flip, data, dim); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 } void fht_kacs_walk(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) fht_kacs_walk_neon(data, len); #else #if defined(__AVX512F__) @@ -101,11 +102,11 @@ void fht_kacs_walk(float *data, size_t len) { } #endif fht_kacs_walk_scalar(data, len); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 } void fht_inv_kacs_walk(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) fht_inv_kacs_walk_neon(data, len); #else #if defined(__AVX512F__) @@ -127,7 +128,7 @@ void fht_inv_kacs_walk(float *data, size_t len) { } #endif fht_inv_kacs_walk_scalar(data, len); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 } void fht_inplace(float *data, size_t n) { diff --git a/src/ailego/math/fht_neon.cc b/src/ailego/math/fht_neon.cc index 6e7064472..7f3ab1499 100644 --- a/src/ailego/math/fht_neon.cc +++ b/src/ailego/math/fht_neon.cc @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if defined(__ARM_NEON) && defined(__aarch64__) +#include + +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) #include #include @@ -40,7 +42,10 @@ void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { uint32_t b1 = (bits16 >> 1) & 1u; uint32_t b2 = (bits16 >> 2) & 1u; uint32_t b3 = (bits16 >> 3) & 1u; - uint32x4_t bit_mask = {b0, b1, b2, b3}; + // Build the lane mask via vld1q_u32: MSVC's uint32x4_t is a union type, + // so GCC/Clang-style brace initialization of a vector is not portable. + const uint32_t bits[4] = {b0, b1, b2, b3}; + uint32x4_t bit_mask = vld1q_u32(bits); uint32x4_t sign_mask = vmulq_u32(bit_mask, sign_bit); float32x4_t v = vld1q_f32(&data[i]); v = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v), sign_mask)); @@ -104,4 +109,4 @@ void fht_inv_kacs_walk_neon(float *data, size_t len) { } // namespace ailego } // namespace zvec -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc index c8f50fb8c..1c54b1f57 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc @@ -63,7 +63,7 @@ float MipsInnerProductSparseInSegment(uint32_t m_sparse_count, //! Compute the distance between matrix and query by SphericalInjection void MipsSquaredEuclideanDistanceMatrix::Compute( const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { -#if __ARM_NEON +#if defined(AILEGO_HAVE_NEON) float u2{0.0f}; float v2{0.0f}; float sum = InnerProductAndSquaredNormFp32NEON(p, q, dim, &u2, &v2); @@ -91,7 +91,7 @@ void MipsSquaredEuclideanDistanceMatrix::Compute( #endif // __SSE__ *out = MipsEuclideanDistanceSphericalInjectionFp32Scalar(p, q, dim, e2); return; -#endif //__ARM_NEON +#endif // AILEGO_HAVE_NEON } //! Compute the distance between matrix and query by RepeatedQuadraticInjection @@ -140,7 +140,7 @@ void MipsSquaredEuclideanDistanceMatrix::Compute( e2); return; -#endif //__ARM_NEON +#endif // AILEGO_HAVE_NEON } // Sparse diff --git a/src/turbo/distance/neon/rotate/fht/fht.cc b/src/turbo/distance/neon/rotate/fht/fht.cc index 0ffbfaad4..c093f691a 100644 --- a/src/turbo/distance/neon/rotate/fht/fht.cc +++ b/src/turbo/distance/neon/rotate/fht/fht.cc @@ -13,7 +13,8 @@ // limitations under the License. #include "fht.h" -#if defined(__ARM_NEON) && defined(__aarch64__) +#include +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) #include #endif #include @@ -26,7 +27,7 @@ namespace zvec::turbo::neon { void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) const uint32x4_t sign_bit = vdupq_n_u32(0x80000000u); size_t simd_end = dim & ~3u; size_t flip_bytes = (dim + 7) / 8; @@ -43,7 +44,10 @@ void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { uint32_t b1 = (bits16 >> 1) & 1u; uint32_t b2 = (bits16 >> 2) & 1u; uint32_t b3 = (bits16 >> 3) & 1u; - uint32x4_t bit_mask = {b0, b1, b2, b3}; + // Build the lane mask via vld1q_u32: MSVC's uint32x4_t is a union type, + // so GCC/Clang-style brace initialization of a vector is not portable. + const uint32_t bits[4] = {b0, b1, b2, b3}; + uint32x4_t bit_mask = vld1q_u32(bits); uint32x4_t sign_mask = vmulq_u32(bit_mask, sign_bit); float32x4_t v = vld1q_f32(&data[i]); v = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v), sign_mask)); @@ -56,14 +60,12 @@ void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { } } #else - (void)flip; - (void)data; - (void)dim; + scalar::fht_flip_sign(flip, data, dim); #endif } void fht_kacs_walk_neon(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) size_t half = len / 2; size_t base = len % 2; size_t offset = base + half; @@ -85,13 +87,12 @@ void fht_kacs_walk_neon(float *data, size_t len) { data[half] *= std::sqrt(2.0f); } #else - (void)data; - (void)len; + scalar::fht_kacs_walk(data, len); #endif } void fht_inv_kacs_walk_neon(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) size_t half = len / 2; size_t base = len % 2; size_t offset = base + half; @@ -114,13 +115,12 @@ void fht_inv_kacs_walk_neon(float *data, size_t len) { data[i + offset] = (a - b) * 0.5f; } #else - (void)data; - (void)len; + scalar::fht_inv_kacs_walk(data, len); #endif } void fht_vec_rescale_neon(float *data, size_t n, float factor) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) const float32x4_t fac = vdupq_n_f32(factor); size_t simd_end = n & ~3u; for (size_t i = 0; i < simd_end; i += 4) { @@ -132,39 +132,37 @@ void fht_vec_rescale_neon(float *data, size_t n, float factor) { data[i] *= factor; } #else - (void)data; - (void)n; - (void)factor; + scalar::fht_vec_rescale(data, n, factor); #endif } void fht_rotate_neon(const float *in, float *out, size_t in_dim, - size_t /*out_dim*/, void *ctx) { -#if defined(__ARM_NEON) && defined(__aarch64__) + size_t out_dim, void *ctx) { +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) + (void)out_dim; static constexpr FhtPrimitives kPrim = { fht_flip_sign_neon, scalar::fht_inplace, fht_kacs_walk_neon, fht_inv_kacs_walk_neon, fht_vec_rescale_neon}; fht_rotate_impl(in, out, in_dim, ctx, kPrim); #else - (void)in; - (void)out; - (void)in_dim; - (void)ctx; + // Without NEON this translation unit still compiles, so delegate to the + // scalar rotator. Never leave `out` unwritten: the runtime dispatcher in + // turbo.cc selects these entry points from CpuFeatures flags, and a no-op + // here would silently emit uninitialized vectors instead of rotated ones. + scalar::fht_rotate(in, out, in_dim, out_dim, ctx); #endif } void fht_unrotate_neon(const float *in, float *out, size_t in_dim, - size_t /*out_dim*/, void *ctx) { -#if defined(__ARM_NEON) && defined(__aarch64__) + size_t out_dim, void *ctx) { +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) + (void)out_dim; static constexpr FhtPrimitives kPrim = { fht_flip_sign_neon, scalar::fht_inplace, fht_kacs_walk_neon, fht_inv_kacs_walk_neon, fht_vec_rescale_neon}; fht_unrotate_impl(in, out, in_dim, ctx, kPrim); #else - (void)in; - (void)out; - (void)in_dim; - (void)ctx; + scalar::fht_unrotate(in, out, in_dim, out_dim, ctx); #endif } From 16aa7e42c1e189fd0d4a6ad78502e7e4da86f9b3 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Thu, 13 Aug 2026 21:04:25 +0200 Subject: [PATCH 7/9] refactor(ailego): add AILEGO_ARM64_GNU_LIKE and refresh guard comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review feedback on #352. `__ARM_NEON && __aarch64__` appeared in ~10 guards meaning "AArch64 NEON under GCC/Clang", which is narrower than `AILEGO_HAVE_NEON && AILEGO_ARM64`: MSVC ARM64 has NEON but exposes neither `float16_t` nor the `v*_f16` intrinsics unless built for ARMv8.2 FP16. Give that condition a name, `AILEGO_ARM64_GNU_LIKE`, and document when to reach for it (FP16 kernels) versus `AILEGO_HAVE_NEON` (FP32 kernels, which MSVC supports). Substitutions are 1:1, so behaviour is unchanged on every target. The inner `#if defined(__ARM_NEON)` branch selectors inside FP16 blocks already gated on `(__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE` are equivalent within that block — `__F16C__` is x86-only and `__ARM_NEON` is ARM-only — so they move to the named macro too. Also refresh the trailing `#endif` comments left naming the old macros after the earlier conversion to `AILEGO_HAVE_NEON` / `AILEGO_ARM64`, which no longer matched the conditions directly above them. Left alone deliberately: the bare `__ARM_NEON` guards in the FP16 distance kernels (`*_fp16_neon.cc`, `*_fp16_dispatch.cc`). Those admit 32-bit ARMv7 as well as AArch64, so narrowing them to `AILEGO_ARM64_GNU_LIKE` would change behaviour on a platform this PR does not target and CI does not cover. They already evaluate false on MSVC ARM64, which is the correct result. Verified on Windows ARM64 (MSVC): `AILEGO_ARM64=1`, `AILEGO_HAVE_NEON=1`, `AILEGO_ARM64_GNU_LIKE=0`, so FP32 NEON kernels compile in while the FP16 ones stay excluded. Touched files compile clean for both ARM64 and x64. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ailego/math/distance_matrix_accum_fp32.i | 4 ++-- src/ailego/math/distance_matrix_fp32.i | 2 +- .../euclidean_distance_matrix_fp32_dispatch.cc | 2 +- .../math/euclidean_distance_matrix_fp32_neon.cc | 2 +- .../math/inner_product_matrix_fp32_dispatch.cc | 4 ++-- src/ailego/math/inner_product_matrix_fp32_neon.cc | 2 +- .../mips_euclidean_distance_matrix_fp16_neon.cc | 4 ++-- .../mips_euclidean_distance_matrix_fp32_neon.cc | 2 +- src/ailego/math/norm1_matrix.h | 7 +++---- src/ailego/math/norm1_matrix_fp16.cc | 7 +++---- src/ailego/math/norm1_matrix_fp32.cc | 2 +- src/ailego/math/norm2_matrix.h | 7 +++---- src/ailego/math/norm2_matrix_fp16.cc | 9 ++++----- src/ailego/math/norm2_matrix_fp32.cc | 2 +- src/ailego/math/normalizer.cc | 15 +++++++-------- src/ailego/math/normalizer.h | 7 +++---- src/ailego/utility/bitset_helper.cc | 4 ++-- src/include/zvec/ailego/internal/platform.h | 11 +++++++++++ 18 files changed, 49 insertions(+), 44 deletions(-) diff --git a/src/ailego/math/distance_matrix_accum_fp32.i b/src/ailego/math/distance_matrix_accum_fp32.i index 7ae6bce2c..8e6c14c79 100644 --- a/src/ailego/math/distance_matrix_accum_fp32.i +++ b/src/ailego/math/distance_matrix_accum_fp32.i @@ -40,13 +40,13 @@ static inline int32_t vaddvq_s32(int32x4_t v) { int32x2_t s = vadd_s32(vget_low_s32(v), vget_high_s32(v)); return vget_lane_s32(vpadd_s32(s, s), 0); } -#endif //__ARM_NEON && !__aarch64__ +#endif // AILEGO_HAVE_NEON && !AILEGO_ARM64 #if defined(AILEGO_ARM64) #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A64 #else #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A32 -#endif // __aarch64__ +#endif // AILEGO_ARM64 //! Compute the distance between matrix and query (FP32, M=2, N=1) #define ACCUM_FP32_2X1_SSE(m, q, dim, out, _NORM) \ diff --git a/src/ailego/math/distance_matrix_fp32.i b/src/ailego/math/distance_matrix_fp32.i index 10cac5e0c..ef13df953 100644 --- a/src/ailego/math/distance_matrix_fp32.i +++ b/src/ailego/math/distance_matrix_fp32.i @@ -28,7 +28,7 @@ #if defined(AILEGO_HAVE_NEON) && !defined(AILEGO_ARM64) #define vdupq_laneq_f32(a, b) vdupq_n_f32(vgetq_lane_f32(a, b)) -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_HAVE_NEON && !AILEGO_ARM64 //! Iterative process of computing distance (FP32, M=2, N=1) #define MATRIX_FP32_ITER_2X1_SSE(m, q, _RES, _LOAD, _PROC) \ diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc index 9dabbe757..8df53ca65 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc @@ -72,7 +72,7 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, } #endif // __SSE__ *out = SquaredEuclideanDistanceFp32Scalar(m, q, dim); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } //----------------------------------------------------------- diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc index 8088bb7e1..d04eaf291 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc @@ -56,7 +56,7 @@ void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, *out = result; } -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } // namespace ailego } // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc index 867b9a3e8..331818dc6 100644 --- a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc +++ b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc @@ -73,7 +73,7 @@ void InnerProductMatrix::Compute(const float *m, const float *q, } #endif // __SSE__ *out = InnerProductFp32Scalar(m, q, dim); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } //! Compute the distance between matrix and query (FP32, M=1, N=1) @@ -104,7 +104,7 @@ void MinusInnerProductMatrix::Compute(const float *m, } #endif // __SSE__ *out = MinusInnerProductFp32Scalar(m, q, dim); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } //-------------------------------------------------- diff --git a/src/ailego/math/inner_product_matrix_fp32_neon.cc b/src/ailego/math/inner_product_matrix_fp32_neon.cc index 4f8028359..fe344abf5 100644 --- a/src/ailego/math/inner_product_matrix_fp32_neon.cc +++ b/src/ailego/math/inner_product_matrix_fp32_neon.cc @@ -59,7 +59,7 @@ float MinusInnerProductFp32NEON(const float *lhs, const float *rhs, return -1 * InnerProductFp32NEON(lhs, rhs, size); } -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc index b4f4c970d..dc31cad50 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_GNU_LIKE) #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) //! Compute the Inner Product between p and q, and each Squared L2-Norm value float InnerProductAndSquaredNormFp16NEON(const Float16 *lhs, const Float16 *rhs, @@ -153,7 +153,7 @@ float MipsEuclideanDistanceRepeatedQuadraticInjectionFp16NEON( return sum; } -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc index 78198ecf1..c6eb428c3 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc @@ -71,7 +71,7 @@ float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, return result; } -#endif //__ARM_NEON +#endif // AILEGO_HAVE_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm1_matrix.h b/src/ailego/math/norm1_matrix.h index 3223e2fca..d18805251 100644 --- a/src/ailego/math/norm1_matrix.h +++ b/src/ailego/math/norm1_matrix.h @@ -127,12 +127,11 @@ struct Norm1Matrix { //! Compute the L1-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON // specialization to gcc/clang aarch64. -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) /*! L1-Norm Matrix (FP16, M=1) */ template <> @@ -143,7 +142,7 @@ struct Norm1Matrix { //! Compute the L1-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm1_matrix_fp16.cc b/src/ailego/math/norm1_matrix_fp16.cc index 0c092bbb7..a05bc5a3f 100644 --- a/src/ailego/math/norm1_matrix_fp16.cc +++ b/src/ailego/math/norm1_matrix_fp16.cc @@ -70,12 +70,11 @@ static const __m512 ABS_MASK_FP32_AVX512 = // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the NEON FP16 // kernel is gated to gcc/clang aarch64. The generic Float16 path is used // on MSVC ARM64. -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) //! Compute the L1-norm of vectors (FP16, M=1) void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NORM_FP16_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) @@ -87,7 +86,7 @@ void Norm1Matrix::Compute(const ValueType *m, size_t dim, NORM_FP16_1_AVX(m, dim, out, ) #endif } -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm1_matrix_fp32.cc b/src/ailego/math/norm1_matrix_fp32.cc index 78b24b0f3..329bcf0c5 100644 --- a/src/ailego/math/norm1_matrix_fp32.cc +++ b/src/ailego/math/norm1_matrix_fp32.cc @@ -78,7 +78,7 @@ void Norm1Matrix::Compute(const ValueType *m, size_t dim, NORM_FP32_1_SSE(m, dim, out, ) #endif } -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm2_matrix.h b/src/ailego/math/norm2_matrix.h index 568fe4151..4359c733c 100644 --- a/src/ailego/math/norm2_matrix.h +++ b/src/ailego/math/norm2_matrix.h @@ -393,12 +393,11 @@ struct SquaredNorm2Matrix { //! Compute the squared L2-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON // specialization to gcc/clang aarch64. -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) /*! L2-Norm Matrix (FP16, M=1) */ template <> @@ -420,7 +419,7 @@ struct SquaredNorm2Matrix { //! Compute the squared L2-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm2_matrix_fp16.cc b/src/ailego/math/norm2_matrix_fp16.cc index a87437d76..259d6e1e9 100644 --- a/src/ailego/math/norm2_matrix_fp16.cc +++ b/src/ailego/math/norm2_matrix_fp16.cc @@ -54,12 +54,11 @@ namespace ailego { // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the NEON FP16 // kernel is gated to gcc/clang aarch64. -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) //! Compute the L2-norm of vectors (FP16, M=1) void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NORM_FP16_1_NEON(m, dim, out, std::sqrt) #else #if defined(__AVX512F__) @@ -75,7 +74,7 @@ void Norm2Matrix::Compute(const ValueType *m, size_t dim, //! Compute the L2-norm of vectors (FP16, M=1) void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NORM_FP16_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) @@ -87,7 +86,7 @@ void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, NORM_FP16_1_AVX(m, dim, out, ) #endif } -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/norm2_matrix_fp32.cc b/src/ailego/math/norm2_matrix_fp32.cc index 2fa59618a..dd6c079b1 100644 --- a/src/ailego/math/norm2_matrix_fp32.cc +++ b/src/ailego/math/norm2_matrix_fp32.cc @@ -87,7 +87,7 @@ void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, NORM_FP32_1_SSE(m, dim, out, ) #endif } -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/normalizer.cc b/src/ailego/math/normalizer.cc index 1a65c41ca..145bdc935 100644 --- a/src/ailego/math/normalizer.cc +++ b/src/ailego/math/normalizer.cc @@ -120,7 +120,7 @@ static inline void NormalizeNEON(float16_t *arr, size_t dim, float norm) { } #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC #endif // !_MSC_VER (FP16 NEON: gcc/clang aarch64 only) -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 #if defined(__AVX__) #if defined(__AVX512F__) @@ -417,19 +417,18 @@ void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { } #endif // __AVX__ NormalizeSSE(arr, dim, norm); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 // `NormalizeNEON` variants are not defined there (see top of file). // Exclude MSVC from this specialization so MSVC ARM64 falls back to the // generic uint16_t-based Float16 normalize path. -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) //! Compute the norm of vector void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NormalizeNEON(reinterpret_cast(arr), dim, norm); #else #if defined(__AVX512F__) @@ -439,9 +438,9 @@ void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { } #endif // __AVX512F__ NormalizeAVX(reinterpret_cast(arr), dim, norm); -#endif // __ARM_NEON +#endif // AILEGO_ARM64_GNU_LIKE } -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/normalizer.h b/src/ailego/math/normalizer.h index 16d9f0050..435c606c2 100644 --- a/src/ailego/math/normalizer.h +++ b/src/ailego/math/normalizer.h @@ -78,12 +78,11 @@ struct Normalizer { } } }; -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 NEON // `Normalizer` specialization is gated to gcc/clang aarch64. -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) /*! Normalizer (FP16) */ template <> @@ -110,7 +109,7 @@ struct Normalizer { } } }; -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/utility/bitset_helper.cc b/src/ailego/utility/bitset_helper.cc index 196678fde..69e5a6d24 100644 --- a/src/ailego/utility/bitset_helper.cc +++ b/src/ailego/utility/bitset_helper.cc @@ -1839,7 +1839,7 @@ static inline size_t bitset_or_cardinality(const uint32_t *lhs, return count; } -#else // !__ARM_NEON && !AILEGO_M64 +#else // !(AILEGO_HAVE_NEON && AILEGO_ARM64) && !AILEGO_M64 static inline size_t bitset_cardinality(const uint32_t *lhs, size_t size) { const uint32_t *last = lhs + size; const uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1964,7 +1964,7 @@ static inline size_t bitset_or_cardinality(const uint32_t *lhs, } return count; } -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 namespace zvec { diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index fac32fed4..d4c53408b 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -23,12 +23,23 @@ // AILEGO_HAVE_NEON - NEON intrinsics available. GCC/Clang predefine // __ARM_NEON; MSVC ARM64 has NEON (ARMv8 baseline) and // predefines only _M_ARM64. +// AILEGO_ARM64_GNU_LIKE +// - AArch64 NEON under a GCC-like compiler (GCC/Clang). +// Narrower than `AILEGO_HAVE_NEON && AILEGO_ARM64`: it +// excludes MSVC ARM64, which does not expose `float16_t` +// or the `v*_f16` intrinsics unless built for ARMv8.2 +// FP16 (`/arch:armv8.2` + `_M_ARM64FP16`). Use this to +// gate FP16 NEON kernels; use AILEGO_HAVE_NEON (and +// AILEGO_ARM64) for FP32 kernels, which MSVC supports. #if defined(__aarch64__) || defined(_M_ARM64) #define AILEGO_ARM64 1 #endif #if defined(__ARM_NEON) || defined(_M_ARM64) #define AILEGO_HAVE_NEON 1 #endif +#if defined(__ARM_NEON) && defined(__aarch64__) +#define AILEGO_ARM64_GNU_LIKE 1 +#endif #if defined(_WIN32) || defined(_WIN64) #include From 55ca580246bbcd21815e01e235498ba743913731 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Sat, 15 Aug 2026 23:37:50 +0200 Subject: [PATCH 8/9] refactor(ailego): name the remaining ARM guard conditions Review follow-ups on #352. Add two macros to platform.h and use them everywhere: - `AILEGO_ARM` for `__arm__ || AILEGO_ARM64`, the `ailego_yield()` guard. - `AILEGO_ARM64_NEON` for `AILEGO_HAVE_NEON && AILEGO_ARM64`, which the FP32 kernels repeated at 16 sites. Switch the `__fp16` paths in float_helper to `AILEGO_ARM64_GNU_LIKE`. The header and the source hold inverse guards and must agree, since they pick `Float16::value_`'s storage type, so float_helper.h now includes platform.h and names the same macro. Also reflow `fht_rotate_neon`'s signature, which drifted past clang-format when `out_dim` was un-commented, and drop the stray CRLF two lines of normalizer.cc picked up from an editor. Both broke the lint job. Substitutions are 1:1 on every supported target. Verified with clang-format 18.1.8, the version CI pins: all 784 checked files clean. On MSVC ARM64 `AILEGO_ARM64`, `AILEGO_ARM`, `AILEGO_HAVE_NEON` and `AILEGO_ARM64_NEON` are set and `AILEGO_ARM64_GNU_LIKE` is not; the turbo rotator's assembly listing carries NEON vector instructions, confirming the kernels compile in rather than falling back. Touched files build clean under /W4 for ARM64 and x64. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ailego/math/fht_dispatch.cc | 14 +++---- src/ailego/math/fht_neon.cc | 4 +- src/ailego/math/norm1_matrix.h | 4 +- src/ailego/math/norm1_matrix_fp32.cc | 4 +- src/ailego/math/norm2_matrix.h | 4 +- src/ailego/math/norm2_matrix_fp32.cc | 4 +- src/ailego/math/normalizer.cc | 8 ++-- src/ailego/math/normalizer.h | 4 +- src/ailego/utility/bitset_helper.cc | 6 +-- src/ailego/utility/float_helper.cc | 11 +++--- src/include/zvec/ailego/internal/platform.h | 37 ++++++++++--------- .../zvec/ailego/utility/float_helper.h | 11 +++--- src/turbo/distance/neon/rotate/fht/fht.cc | 18 ++++----- 13 files changed, 65 insertions(+), 64 deletions(-) diff --git a/src/ailego/math/fht_dispatch.cc b/src/ailego/math/fht_dispatch.cc index 1c1cea834..00da277ba 100644 --- a/src/ailego/math/fht_dispatch.cc +++ b/src/ailego/math/fht_dispatch.cc @@ -42,7 +42,7 @@ void fht_kacs_walk_avx512(float *data, size_t len); void fht_inv_kacs_walk_avx512(float *data, size_t len); void fht_inplace_avx512(float *data, size_t n); #endif -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim); void fht_kacs_walk_neon(float *data, size_t len); void fht_inv_kacs_walk_neon(float *data, size_t len); @@ -53,7 +53,7 @@ void fht_inv_kacs_walk_neon(float *data, size_t len); // ============================================================================ void fht_flip_sign(const uint8_t *flip, float *data, size_t dim) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) fht_flip_sign_neon(flip, data, dim); #else #if defined(__AVX512F__) @@ -76,11 +76,11 @@ void fht_flip_sign(const uint8_t *flip, float *data, size_t dim) { } #endif fht_flip_sign_scalar(flip, data, dim); -#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 +#endif // AILEGO_ARM64_NEON } void fht_kacs_walk(float *data, size_t len) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) fht_kacs_walk_neon(data, len); #else #if defined(__AVX512F__) @@ -102,11 +102,11 @@ void fht_kacs_walk(float *data, size_t len) { } #endif fht_kacs_walk_scalar(data, len); -#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 +#endif // AILEGO_ARM64_NEON } void fht_inv_kacs_walk(float *data, size_t len) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) fht_inv_kacs_walk_neon(data, len); #else #if defined(__AVX512F__) @@ -128,7 +128,7 @@ void fht_inv_kacs_walk(float *data, size_t len) { } #endif fht_inv_kacs_walk_scalar(data, len); -#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 +#endif // AILEGO_ARM64_NEON } void fht_inplace(float *data, size_t n) { diff --git a/src/ailego/math/fht_neon.cc b/src/ailego/math/fht_neon.cc index 7f3ab1499..22b3f112e 100644 --- a/src/ailego/math/fht_neon.cc +++ b/src/ailego/math/fht_neon.cc @@ -14,7 +14,7 @@ #include -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) #include #include @@ -109,4 +109,4 @@ void fht_inv_kacs_walk_neon(float *data, size_t len) { } // namespace ailego } // namespace zvec -#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 +#endif // AILEGO_ARM64_NEON diff --git a/src/ailego/math/norm1_matrix.h b/src/ailego/math/norm1_matrix.h index d18805251..fd42c389e 100644 --- a/src/ailego/math/norm1_matrix.h +++ b/src/ailego/math/norm1_matrix.h @@ -116,7 +116,7 @@ struct Norm1Matrix< } }; -#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) /*! L1-Norm Matrix (FP32, M=1) */ template <> @@ -127,7 +127,7 @@ struct Norm1Matrix { //! Compute the L1-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) +#endif // __SSE__ || AILEGO_ARM64_NEON // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON // specialization to gcc/clang aarch64. diff --git a/src/ailego/math/norm1_matrix_fp32.cc b/src/ailego/math/norm1_matrix_fp32.cc index 329bcf0c5..e988d7411 100644 --- a/src/ailego/math/norm1_matrix_fp32.cc +++ b/src/ailego/math/norm1_matrix_fp32.cc @@ -56,7 +56,7 @@ namespace ailego { //! Calculate sum of absolute (NEON) #define SA_FP32_NEON(v_m, v_sum) v_sum = vaddq_f32(vabsq_f32(v_m), v_sum); -#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) //! Compute the L1-norm of vectors (FP32, M=1) void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { @@ -78,7 +78,7 @@ void Norm1Matrix::Compute(const ValueType *m, size_t dim, NORM_FP32_1_SSE(m, dim, out, ) #endif } -#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) +#endif // __SSE__ || AILEGO_ARM64_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm2_matrix.h b/src/ailego/math/norm2_matrix.h index 4359c733c..05d7890b4 100644 --- a/src/ailego/math/norm2_matrix.h +++ b/src/ailego/math/norm2_matrix.h @@ -371,7 +371,7 @@ struct SquaredNorm2Matrix= 2>::type> { } }; -#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) /*! L2-Norm Matrix (FP32, M=1) */ template <> @@ -393,7 +393,7 @@ struct SquaredNorm2Matrix { //! Compute the squared L2-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) +#endif // __SSE__ || AILEGO_ARM64_NEON // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON // specialization to gcc/clang aarch64. diff --git a/src/ailego/math/norm2_matrix_fp32.cc b/src/ailego/math/norm2_matrix_fp32.cc index dd6c079b1..8124255f0 100644 --- a/src/ailego/math/norm2_matrix_fp32.cc +++ b/src/ailego/math/norm2_matrix_fp32.cc @@ -43,7 +43,7 @@ namespace ailego { //! Calculate sum of squared (NEON) #define SS_FP32_NEON(v_m, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_m); -#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) //! Compute the L2-norm of vectors (FP32, M=1) void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { @@ -87,7 +87,7 @@ void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, NORM_FP32_1_SSE(m, dim, out, ) #endif } -#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) +#endif // __SSE__ || AILEGO_ARM64_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/normalizer.cc b/src/ailego/math/normalizer.cc index 145bdc935..239282e0f 100644 --- a/src/ailego/math/normalizer.cc +++ b/src/ailego/math/normalizer.cc @@ -17,7 +17,7 @@ namespace zvec { namespace ailego { -#if (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(AILEGO_ARM64_NEON) static inline void NormalizeNEON(float *arr, size_t dim, float norm) { float *last = arr + dim; float *last_aligned = arr + ((dim >> 3) << 3); @@ -120,7 +120,7 @@ static inline void NormalizeNEON(float16_t *arr, size_t dim, float norm) { } #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC #endif // !_MSC_VER (FP16 NEON: gcc/clang aarch64 only) -#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 +#endif // AILEGO_ARM64_NEON #if defined(__AVX__) #if defined(__AVX512F__) @@ -398,7 +398,7 @@ static inline void NormalizeSSE(float *arr, size_t dim, float norm) { } #endif // __SSE__ -#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) //! Compute the norm of vector void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { #if defined(AILEGO_HAVE_NEON) @@ -419,7 +419,7 @@ void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { NormalizeSSE(arr, dim, norm); #endif // AILEGO_HAVE_NEON } -#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) +#endif // __SSE__ || AILEGO_ARM64_NEON // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 // `NormalizeNEON` variants are not defined there (see top of file). diff --git a/src/ailego/math/normalizer.h b/src/ailego/math/normalizer.h index 435c606c2..996a6a39f 100644 --- a/src/ailego/math/normalizer.h +++ b/src/ailego/math/normalizer.h @@ -51,7 +51,7 @@ struct Normalizer { } }; -#if defined(__SSE__) || (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) /*! Normalizer (FP32) */ template <> @@ -78,7 +78,7 @@ struct Normalizer { } } }; -#endif // __SSE__ || (AILEGO_HAVE_NEON && AILEGO_ARM64) +#endif // __SSE__ || AILEGO_ARM64_NEON // MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 NEON // `Normalizer` specialization is gated to gcc/clang aarch64. diff --git a/src/ailego/utility/bitset_helper.cc b/src/ailego/utility/bitset_helper.cc index 69e5a6d24..95912413d 100644 --- a/src/ailego/utility/bitset_helper.cc +++ b/src/ailego/utility/bitset_helper.cc @@ -1480,7 +1480,7 @@ static inline bool bitset_test_none(const uint32_t *lhs, size_t size) { #endif // AILEGO_M64 #endif // __AVX2__ -#if (defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64)) +#if defined(AILEGO_ARM64_NEON) static inline size_t bitset_cardinality(const uint32_t *lhs, size_t size) { const uint32_t *last = lhs + size; const uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1839,7 +1839,7 @@ static inline size_t bitset_or_cardinality(const uint32_t *lhs, return count; } -#else // !(AILEGO_HAVE_NEON && AILEGO_ARM64) && !AILEGO_M64 +#else // !AILEGO_ARM64_NEON && !AILEGO_M64 static inline size_t bitset_cardinality(const uint32_t *lhs, size_t size) { const uint32_t *last = lhs + size; const uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1964,7 +1964,7 @@ static inline size_t bitset_or_cardinality(const uint32_t *lhs, } return count; } -#endif // AILEGO_HAVE_NEON && AILEGO_ARM64 +#endif // AILEGO_ARM64_NEON namespace zvec { diff --git a/src/ailego/utility/float_helper.cc b/src/ailego/utility/float_helper.cc index 90ecb92e8..78e5a700e 100644 --- a/src/ailego/utility/float_helper.cc +++ b/src/ailego/utility/float_helper.cc @@ -22,10 +22,9 @@ // #define float32(x) _cvtsh_ss(x) // #endif // __F16C__ && __AVX__ -// MSVC ARM64 lacks the GCC/Clang `__fp16` extension type, so keep this -// path gated on `__aarch64__` (predefined only by GCC/Clang on AArch64). -// MSVC ARM64 falls through to the F16C/scalar paths below. -#if defined(__aarch64__) && !defined(_MSC_VER) +// `__fp16` is a GCC/Clang extension; MSVC ARM64 has no equivalent and +// falls through to the F16C/scalar paths below. +#if defined(AILEGO_ARM64_GNU_LIKE) static inline float float32(uint16_t val) { __fp16 f; memcpy(&f, &val, sizeof(val)); @@ -66,7 +65,7 @@ static inline void convert_fp32_to_fp16(const float *arr, size_t size, out[i] = float16(arr[i] / norm); } } -#else +#else // !AILEGO_ARM64_GNU_LIKE // Refer: https://github.com/Maratyszcza/FP16/blob/master/third-party/half.hpp static inline float float32(uint16_t val) { static const uint32_t mantissa_table[2048] = { @@ -1214,7 +1213,7 @@ static inline void convert_fp32_to_fp16(const float *arr, size_t size, return convert_fp32_to_fp16_fallback(arr, size, norm, out); } -#endif // +#endif // AILEGO_ARM64_GNU_LIKE namespace zvec { namespace ailego { diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index d4c53408b..b43b705d8 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -14,29 +14,30 @@ #pragma once -// Architecture / SIMD feature detection, defined once so the guards below -// (and across the codebase) read clearly instead of repeating long -// compiler-macro disjunctions: +// Architecture / SIMD feature detection, named once so guards across the +// codebase read clearly instead of repeating compiler-macro disjunctions: // -// AILEGO_ARM64 - 64-bit ARM (AArch64). GCC/Clang predefine __aarch64__; -// MSVC predefines _M_ARM64. -// AILEGO_HAVE_NEON - NEON intrinsics available. GCC/Clang predefine -// __ARM_NEON; MSVC ARM64 has NEON (ARMv8 baseline) and -// predefines only _M_ARM64. -// AILEGO_ARM64_GNU_LIKE -// - AArch64 NEON under a GCC-like compiler (GCC/Clang). -// Narrower than `AILEGO_HAVE_NEON && AILEGO_ARM64`: it -// excludes MSVC ARM64, which does not expose `float16_t` -// or the `v*_f16` intrinsics unless built for ARMv8.2 -// FP16 (`/arch:armv8.2` + `_M_ARM64FP16`). Use this to -// gate FP16 NEON kernels; use AILEGO_HAVE_NEON (and -// AILEGO_ARM64) for FP32 kernels, which MSVC supports. +// AILEGO_ARM64 - AArch64. GCC/Clang: __aarch64__; MSVC: _M_ARM64. +// AILEGO_ARM - any ARM, 32- or 64-bit. +// AILEGO_HAVE_NEON - NEON intrinsics available. NEON is an ARMv8 +// baseline, so MSVC ARM64 qualifies. +// AILEGO_ARM64_NEON - AArch64 with NEON. Use for FP32 kernels. +// AILEGO_ARM64_GNU_LIKE - AArch64 NEON under GCC/Clang. Excludes MSVC, +// which exposes neither float16_t nor the v*_f16 +// intrinsics outside ARMv8.2 FP16. Use for FP16 +// kernels and other GCC/Clang-only extensions. #if defined(__aarch64__) || defined(_M_ARM64) #define AILEGO_ARM64 1 #endif +#if defined(__arm__) || defined(AILEGO_ARM64) +#define AILEGO_ARM 1 +#endif #if defined(__ARM_NEON) || defined(_M_ARM64) #define AILEGO_HAVE_NEON 1 #endif +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#define AILEGO_ARM64_NEON 1 +#endif #if defined(__ARM_NEON) && defined(__aarch64__) #define AILEGO_ARM64_GNU_LIKE 1 #endif @@ -266,7 +267,7 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_popcount ailego_popcount32 #endif // AILEGO_M64 -#if defined(__arm__) || defined(AILEGO_ARM64) +#if defined(AILEGO_ARM) // ARMv7 Architecture Reference Manual (for YIELD) // ARM Compiler toolchain Compiler Reference (for __yield() instrinsic) #if defined(__CC_ARM) || defined(_MSC_VER) @@ -280,7 +281,7 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_yield() _mm_pause() #else #define ailego_yield() ((void)0) -#endif // __arm__ || __aarch64__ +#endif // AILEGO_ARM #if defined(_MSC_VER) #define ailego_aligned_malloc(SIZE, ALIGN) \ diff --git a/src/include/zvec/ailego/utility/float_helper.h b/src/include/zvec/ailego/utility/float_helper.h index f9fb13aab..9d04e38a4 100644 --- a/src/include/zvec/ailego/utility/float_helper.h +++ b/src/include/zvec/ailego/utility/float_helper.h @@ -16,6 +16,7 @@ #include #include +#include #include namespace zvec { @@ -53,10 +54,10 @@ struct ZVEC_AILEGO_API FloatHelper { } }; -// The `#else` branch below stores `Float16::value_` as `__fp16` — a GCC/Clang -// extension type that MSVC does not provide even on ARM64. Keep the uint16_t -// storage path for MSVC (including MSVC ARM64) so the wrapper compiles. -#if !defined(__aarch64__) || defined(_MSC_VER) +// The `#else` branch below stores `Float16::value_` as `__fp16`, a GCC/Clang +// extension MSVC lacks even on ARM64. Keep the uint16_t storage path there so +// the wrapper still compiles. Must match the guard in float_helper.cc. +#if !defined(AILEGO_ARM64_GNU_LIKE) /*! Half-Precision Floating Point */ class Float16 { @@ -232,7 +233,7 @@ class Float16 { private: __fp16 value_; }; -#endif +#endif // !AILEGO_ARM64_GNU_LIKE // Check size of Float16 static_assert(sizeof(Float16) == 2, "Float16 must be aligned with 2 bytes"); diff --git a/src/turbo/distance/neon/rotate/fht/fht.cc b/src/turbo/distance/neon/rotate/fht/fht.cc index c093f691a..03419eb33 100644 --- a/src/turbo/distance/neon/rotate/fht/fht.cc +++ b/src/turbo/distance/neon/rotate/fht/fht.cc @@ -14,7 +14,7 @@ #include "fht.h" #include -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) #include #endif #include @@ -27,7 +27,7 @@ namespace zvec::turbo::neon { void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) const uint32x4_t sign_bit = vdupq_n_u32(0x80000000u); size_t simd_end = dim & ~3u; size_t flip_bytes = (dim + 7) / 8; @@ -65,7 +65,7 @@ void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { } void fht_kacs_walk_neon(float *data, size_t len) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) size_t half = len / 2; size_t base = len % 2; size_t offset = base + half; @@ -92,7 +92,7 @@ void fht_kacs_walk_neon(float *data, size_t len) { } void fht_inv_kacs_walk_neon(float *data, size_t len) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) size_t half = len / 2; size_t base = len % 2; size_t offset = base + half; @@ -120,7 +120,7 @@ void fht_inv_kacs_walk_neon(float *data, size_t len) { } void fht_vec_rescale_neon(float *data, size_t n, float factor) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) const float32x4_t fac = vdupq_n_f32(factor); size_t simd_end = n & ~3u; for (size_t i = 0; i < simd_end; i += 4) { @@ -136,9 +136,9 @@ void fht_vec_rescale_neon(float *data, size_t n, float factor) { #endif } -void fht_rotate_neon(const float *in, float *out, size_t in_dim, - size_t out_dim, void *ctx) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +void fht_rotate_neon(const float *in, float *out, size_t in_dim, size_t out_dim, + void *ctx) { +#if defined(AILEGO_ARM64_NEON) (void)out_dim; static constexpr FhtPrimitives kPrim = { fht_flip_sign_neon, scalar::fht_inplace, fht_kacs_walk_neon, @@ -155,7 +155,7 @@ void fht_rotate_neon(const float *in, float *out, size_t in_dim, void fht_unrotate_neon(const float *in, float *out, size_t in_dim, size_t out_dim, void *ctx) { -#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#if defined(AILEGO_ARM64_NEON) (void)out_dim; static constexpr FhtPrimitives kPrim = { fht_flip_sign_neon, scalar::fht_inplace, fht_kacs_walk_neon, From 0a75d487aca248de217d455226320a5962139c44 Mon Sep 17 00:00:00 2001 From: Andreas Martin Aanerud Date: Mon, 17 Aug 2026 07:31:41 +0200 Subject: [PATCH 9/9] fix(turbo,ailego): repair NEON PQ kernels and SSE FHT linkage on MSVC Windows CI exposed two defects after CpuFeatures::NEON() began reporting true on MSVC ARM64. neon/pq_quantizer_int8/pq_distance.cc guarded on __ARM_NEON && __aarch64__, which MSVC never defines, and its #else branches returned without writing *out. Enabling NEON selected those stubs, so the ADC tests read uninitialised memory. The guards now use AILEGO_ARM64_NEON and the fallbacks delegate to the scalar kernels. pq_adc_int8_batch_distance_neon also scored every lane against chunk m's sub-table instead of chunks m..m+3, so results diverged from scalar once num_chunk reached 4. Each lane now loads its own table. A scalar-equivalence harness over 363 shape and value combinations reports no mismatches. fht_sse.cc, fht_avx2.cc and fht_avx512.cc test __SSE2__/__AVX2__/__AVX512F__ but did not include platform.h, which synthesises those macros for MSVC. fht_dispatch.cc did include it, so it called SSE entry points that compiled away to nothing (LNK2019). Including platform.h in the ISA files restores agreement and gives Windows x64 the SSE FHT path Linux x64 already uses. MSVC models NEON vector types as unions, so the GCC-style brace initialisers were replaced with vld1q_f32 over plain arrays. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/ailego/math/fht_avx2.cc | 2 + src/ailego/math/fht_avx512.cc | 2 + src/ailego/math/fht_sse.cc | 2 + .../neon/pq_quantizer_int8/pq_distance.cc | 69 ++++++++++--------- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/ailego/math/fht_avx2.cc b/src/ailego/math/fht_avx2.cc index b2a7f91f9..d985e73a0 100644 --- a/src/ailego/math/fht_avx2.cc +++ b/src/ailego/math/fht_avx2.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #if defined(__AVX2__) #include diff --git a/src/ailego/math/fht_avx512.cc b/src/ailego/math/fht_avx512.cc index ba0fa32d7..0033f709c 100644 --- a/src/ailego/math/fht_avx512.cc +++ b/src/ailego/math/fht_avx512.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #if defined(__AVX512F__) #include diff --git a/src/ailego/math/fht_sse.cc b/src/ailego/math/fht_sse.cc index 1c03b1625..8aa4e743a 100644 --- a/src/ailego/math/fht_sse.cc +++ b/src/ailego/math/fht_sse.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #if defined(__SSE2__) #include diff --git a/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc b/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc index 24cacc7f8..5df1567eb 100644 --- a/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc +++ b/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc @@ -18,15 +18,17 @@ // the running accumulator, halving the number of FP add operations. #include "neon/pq_quantizer_int8/pq_distance.h" -#if defined(__ARM_NEON) && defined(__aarch64__) +#include +#if defined(AILEGO_ARM64_NEON) #include #endif #include #include +#include "scalar/pq_quantizer_int8/pq_distance.h" namespace zvec::turbo::neon { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) namespace { // Horizontal sum of 4 floats in a float32x4_t register via pairwise add. @@ -43,7 +45,7 @@ inline float horizontal_sum_neon(float32x4_t v) { void pq_adc_int8_distance_neon(const void *pq_code_v, const void *lut_v, size_t num_chunk, float *out) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) constexpr int kNumCentroids = 256; constexpr int kChunkSize = 4; // NEON processes 4 floats at once const auto *pq_code = reinterpret_cast(pq_code_v); @@ -60,7 +62,8 @@ void pq_adc_int8_distance_neon(const void *pq_code_v, const void *lut_v, float d1 = lut[(m + 1) * kNumCentroids + pq_code[m + 1]]; float d2 = lut[(m + 2) * kNumCentroids + pq_code[m + 2]]; float d3 = lut[(m + 3) * kNumCentroids + pq_code[m + 3]]; - float32x4_t d = {d0, d1, d2, d3}; + const float lane[4] = {d0, d1, d2, d3}; + float32x4_t d = vld1q_f32(lane); acc = vaddq_f32(acc, d); } @@ -73,17 +76,18 @@ void pq_adc_int8_distance_neon(const void *pq_code_v, const void *lut_v, *out = sum; #else - (void)pq_code_v; - (void)lut_v; - (void)num_chunk; - (void)out; + // Without NEON this translation unit still compiles, so delegate to the + // scalar kernel. Never leave `out` unwritten: turbo.cc selects these entry + // points from CpuFeatures flags, and a no-op here would silently return + // whatever the caller's buffer already held. + scalar::pq_adc_int8_distance(pq_code_v, lut_v, num_chunk, out); #endif } void pq_sdc_int8_distance_neon(const void *a_v, const void *b_v, const void *dist_table_v, size_t num_chunk, float *out) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) constexpr int kNumCentroids = 256; constexpr int chunk = kNumCentroids * kNumCentroids; // 65536 constexpr int kChunkSize = 4; @@ -109,7 +113,8 @@ void pq_sdc_int8_distance_neon(const void *a_v, const void *b_v, float d3 = dist_table[(m + 3) * chunk + static_cast(a[m + 3]) * kNumCentroids + static_cast(b[m + 3])]; - float32x4_t d = {d0, d1, d2, d3}; + const float lane[4] = {d0, d1, d2, d3}; + float32x4_t d = vld1q_f32(lane); acc = vaddq_f32(acc, d); } @@ -124,18 +129,14 @@ void pq_sdc_int8_distance_neon(const void *a_v, const void *b_v, *out = sum; #else - (void)a_v; - (void)b_v; - (void)dist_table_v; - (void)num_chunk; - (void)out; + scalar::pq_sdc_int8_distance(a_v, b_v, dist_table_v, num_chunk, out); #endif } void pq_adc_int8_batch_distance_neon(const void **candidates_v, const void *lut_v, size_t num, size_t num_chunk, float *out) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) constexpr int kNumCentroids = 256; constexpr int kChunkSize = 4; constexpr int kBatch = 4; @@ -156,16 +157,26 @@ void pq_adc_int8_batch_distance_neon(const void **candidates_v, size_t m = 0; for (; m + kChunkSize <= num_chunk; m += kChunkSize) { - const float *tab = lut + m * kNumCentroids; - - float32x4_t d0 = {tab[c0[m + 0]], tab[c0[m + 1]], tab[c0[m + 2]], - tab[c0[m + 3]]}; - float32x4_t d1 = {tab[c1[m + 0]], tab[c1[m + 1]], tab[c1[m + 2]], - tab[c1[m + 3]]}; - float32x4_t d2 = {tab[c2[m + 0]], tab[c2[m + 1]], tab[c2[m + 2]], - tab[c2[m + 3]]}; - float32x4_t d3 = {tab[c3[m + 0]], tab[c3[m + 1]], tab[c3[m + 2]], - tab[c3[m + 3]]}; + // Each lane holds a different chunk, so every lane needs its own + // sub-table: lane j reads chunk (m + j). Sharing one base here would + // score every lane against chunk m's centroids. + const float *t0 = lut + (m + 0) * kNumCentroids; + const float *t1 = lut + (m + 1) * kNumCentroids; + const float *t2 = lut + (m + 2) * kNumCentroids; + const float *t3 = lut + (m + 3) * kNumCentroids; + + const float lane0[4] = {t0[c0[m + 0]], t1[c0[m + 1]], t2[c0[m + 2]], + t3[c0[m + 3]]}; + const float lane1[4] = {t0[c1[m + 0]], t1[c1[m + 1]], t2[c1[m + 2]], + t3[c1[m + 3]]}; + const float lane2[4] = {t0[c2[m + 0]], t1[c2[m + 1]], t2[c2[m + 2]], + t3[c2[m + 3]]}; + const float lane3[4] = {t0[c3[m + 0]], t1[c3[m + 1]], t2[c3[m + 2]], + t3[c3[m + 3]]}; + float32x4_t d0 = vld1q_f32(lane0); + float32x4_t d1 = vld1q_f32(lane1); + float32x4_t d2 = vld1q_f32(lane2); + float32x4_t d3 = vld1q_f32(lane3); acc0 = vaddq_f32(acc0, d0); acc1 = vaddq_f32(acc1, d1); @@ -196,11 +207,7 @@ void pq_adc_int8_batch_distance_neon(const void **candidates_v, pq_adc_int8_distance_neon(candidates[i], lut, num_chunk, out + i); } #else - (void)candidates_v; - (void)lut_v; - (void)num; - (void)num_chunk; - (void)out; + scalar::pq_adc_int8_batch_distance(candidates_v, lut_v, num, num_chunk, out); #endif }