diff --git a/.github/workflows/05-windows-build.yml b/.github/workflows/05-windows-build.yml index ac59c2f3b..d3bfd0d52 100644 --- a/.github/workflows/05-windows-build.yml +++ b/.github/workflows/05-windows-build.yml @@ -18,7 +18,16 @@ jobs: matrix: include: - platform: windows-2022 + msvc_arch: x64 + python_version: '3.10' - platform: windows-2025 + msvc_arch: x64 + python_version: '3.10' + # Windows ARM64: Python 3.10 has no official ARM64 installer; + # 3.11 is the first CPython release with a Windows-on-ARM build. + - platform: windows-11-arm + msvc_arch: arm64 + python_version: '3.11' env: SCCACHE_GHA_ENABLED: "true" @@ -47,14 +56,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v7 with: - python-version: '3.10' + python-version: ${{ matrix.python_version }} cache: 'pip' cache-dependency-path: 'pyproject.toml' - name: Set up MSVC environment uses: ilammy/msvc-dev-cmd@v1.13.0 with: - arch: x64 + arch: ${{ matrix.msvc_arch }} - name: Set up environment variables run: | diff --git a/src/ailego/CMakeLists.txt b/src/ailego/CMakeLists.txt index f19ebe50f..5656de574 100644 --- a/src/ailego/CMakeLists.txt +++ b/src/ailego/CMakeLists.txt @@ -90,29 +90,35 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) ) endforeach() elseif (HOST_ARCH MATCHES "^(arm|arm64)$") - if(MSVC) - return() - endif() - set(MATH_MARCH_FLAG_NEON "-march=armv8-a") - - file(GLOB_RECURSE MATH_FILES_NEON - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.c - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.cc - ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.c - ) + if(NOT MSVC) + set(MATH_MARCH_FLAG_NEON "-march=armv8-a") + + file(GLOB_RECURSE MATH_FILES_NEON + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.c + ) - foreach(MATH_FILE ${MATH_FILES_NEON}) - set_source_files_properties( - ${MATH_FILE} - PROPERTIES - COMPILE_FLAGS "${MATH_MARCH_FLAG_NEON}" - ) - endforeach() + foreach(MATH_FILE ${MATH_FILES_NEON}) + set_source_files_properties( + ${MATH_FILE} + PROPERTIES + COMPILE_FLAGS "${MATH_MARCH_FLAG_NEON}" + ) + endforeach() + else() + # MSVC on ARM64: NEON is the ARMv8 baseline and is always enabled, + # so no `-march` flag is required (MSVC does not accept GCC-style + # `-march=` anyway). The NEON math kernels still get compiled via + # the ALL_SRCS glob above; their guards use the AILEGO_HAVE_NEON / + # AILEGO_ARM64 feature macros (defined in platform.h), which resolve + # to true on MSVC ARM64 so the bodies actually emit code. + endif() endif() endif() diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index b3e791c95..2173a43d7 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -14,12 +14,11 @@ #include "cpu_features.h" #include +#include -#if defined(_MSC_VER) +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) #include -#endif - -#if (defined(__x86_64__) || defined(__i386__)) && !defined(_MSC_VER) +#elif defined(__x86_64__) || defined(__i386__) #include #endif @@ -43,7 +42,7 @@ namespace internal { CpuFeatures::CpuFlags CpuFeatures::flags_; -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) CpuFeatures::CpuFlags::CpuFlags(void) : L1_ECX(0), L1_EDX(0), L7_EBX(0), L7_ECX(0), L7_EDX(0) { int l1[4] = {0, 0, 0, 0}; @@ -347,7 +346,11 @@ bool CpuFeatures::HYPERVISOR(void) { bool CpuFeatures::NEON(void) { #if defined(__aarch64__) && defined(__linux__) return !!(getauxval(AT_HWCAP) & HWCAP_ASIMD); -#elif defined(__ARM_NEON) +#elif defined(AILEGO_HAVE_NEON) + // NEON is part of the ARMv8 baseline, so it is unconditionally present when + // the target has it (including MSVC ARM64, which predefines only _M_ARM64). + // This must agree with Intrinsics()/version.i, which report "Neon" from the + // same macro. return true; #else return false; @@ -356,7 +359,7 @@ bool CpuFeatures::NEON(void) { const char *CpuFeatures::Intrinsics(void) { return "" -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) "Neon" #if defined(__ARM_FEATURE_CRC32) "+CRC" diff --git a/src/ailego/math/distance_matrix_accum_fp32.i b/src/ailego/math/distance_matrix_accum_fp32.i index c186492c6..8e6c14c79 100644 --- a/src/ailego/math/distance_matrix_accum_fp32.i +++ b/src/ailego/math/distance_matrix_accum_fp32.i @@ -30,7 +30,7 @@ _mm512_castps_si512(b))) #endif // __AVX512DQ__ -#if defined(__ARM_NEON) && !defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && !defined(AILEGO_ARM64) static inline float32_t vaddvq_f32(float32x4_t v) { float32x2_t s = vadd_f32(vget_low_f32(v), vget_high_f32(v)); return vget_lane_f32(vpadd_f32(s, s), 0); @@ -40,13 +40,13 @@ static inline int32_t vaddvq_s32(int32x4_t v) { int32x2_t s = vadd_s32(vget_low_s32(v), vget_high_s32(v)); return vget_lane_s32(vpadd_s32(s, s), 0); } -#endif //__ARM_NEON && !__aarch64__ +#endif // AILEGO_HAVE_NEON && !AILEGO_ARM64 -#if defined(__aarch64__) +#if defined(AILEGO_ARM64) #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A64 #else #define ACCUM_FP32_2X1_NEON ACCUM_FP32_2X1_NEON_A32 -#endif // __aarch64__ +#endif // AILEGO_ARM64 //! Compute the distance between matrix and query (FP32, M=2, N=1) #define ACCUM_FP32_2X1_SSE(m, q, dim, out, _NORM) \ diff --git a/src/ailego/math/distance_matrix_fp32.i b/src/ailego/math/distance_matrix_fp32.i index a9ddcd075..ef13df953 100644 --- a/src/ailego/math/distance_matrix_fp32.i +++ b/src/ailego/math/distance_matrix_fp32.i @@ -26,9 +26,9 @@ _mm256_insertf128_ps(_mm256_castps128_ps256(b), (a), 1) #endif // __AVX__ -#if defined(__ARM_NEON) && !defined(__aarch64__) +#if defined(AILEGO_HAVE_NEON) && !defined(AILEGO_ARM64) #define vdupq_laneq_f32(a, b) vdupq_n_f32(vgetq_lane_f32(a, b)) -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_HAVE_NEON && !AILEGO_ARM64 //! Iterative process of computing distance (FP32, M=2, N=1) #define MATRIX_FP32_ITER_2X1_SSE(m, q, _RES, _LOAD, _PROC) \ diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc index cc3044389..8df53ca65 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc @@ -18,7 +18,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, size_t size, float *out); #endif @@ -49,7 +49,7 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) SquaredEuclideanDistanceFp32NEON(m, q, dim, out); #else #if defined(__AVX512F__) @@ -72,7 +72,7 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, } #endif // __SSE__ *out = SquaredEuclideanDistanceFp32Scalar(m, q, dim); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } //----------------------------------------------------------- diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc index aa1694e21..d04eaf291 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) //! Squared Euclidean Distance void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, size_t size, float *out) { @@ -56,7 +56,7 @@ void SquaredEuclideanDistanceFp32NEON(const float *lhs, const float *rhs, *out = result; } -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } // namespace ailego } // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/fht_avx2.cc b/src/ailego/math/fht_avx2.cc index b2a7f91f9..d985e73a0 100644 --- a/src/ailego/math/fht_avx2.cc +++ b/src/ailego/math/fht_avx2.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #if defined(__AVX2__) #include diff --git a/src/ailego/math/fht_avx512.cc b/src/ailego/math/fht_avx512.cc index ba0fa32d7..0033f709c 100644 --- a/src/ailego/math/fht_avx512.cc +++ b/src/ailego/math/fht_avx512.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #if defined(__AVX512F__) #include diff --git a/src/ailego/math/fht_dispatch.cc b/src/ailego/math/fht_dispatch.cc index b1e9c4aac..00da277ba 100644 --- a/src/ailego/math/fht_dispatch.cc +++ b/src/ailego/math/fht_dispatch.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include "fht.h" namespace zvec { @@ -41,7 +42,7 @@ void fht_kacs_walk_avx512(float *data, size_t len); void fht_inv_kacs_walk_avx512(float *data, size_t len); void fht_inplace_avx512(float *data, size_t n); #endif -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim); void fht_kacs_walk_neon(float *data, size_t len); void fht_inv_kacs_walk_neon(float *data, size_t len); @@ -52,7 +53,7 @@ void fht_inv_kacs_walk_neon(float *data, size_t len); // ============================================================================ void fht_flip_sign(const uint8_t *flip, float *data, size_t dim) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) fht_flip_sign_neon(flip, data, dim); #else #if defined(__AVX512F__) @@ -75,11 +76,11 @@ void fht_flip_sign(const uint8_t *flip, float *data, size_t dim) { } #endif fht_flip_sign_scalar(flip, data, dim); -#endif // __ARM_NEON +#endif // AILEGO_ARM64_NEON } void fht_kacs_walk(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) fht_kacs_walk_neon(data, len); #else #if defined(__AVX512F__) @@ -101,11 +102,11 @@ void fht_kacs_walk(float *data, size_t len) { } #endif fht_kacs_walk_scalar(data, len); -#endif // __ARM_NEON +#endif // AILEGO_ARM64_NEON } void fht_inv_kacs_walk(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) fht_inv_kacs_walk_neon(data, len); #else #if defined(__AVX512F__) @@ -127,7 +128,7 @@ void fht_inv_kacs_walk(float *data, size_t len) { } #endif fht_inv_kacs_walk_scalar(data, len); -#endif // __ARM_NEON +#endif // AILEGO_ARM64_NEON } void fht_inplace(float *data, size_t n) { diff --git a/src/ailego/math/fht_neon.cc b/src/ailego/math/fht_neon.cc index 6e7064472..22b3f112e 100644 --- a/src/ailego/math/fht_neon.cc +++ b/src/ailego/math/fht_neon.cc @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if defined(__ARM_NEON) && defined(__aarch64__) +#include + +#if defined(AILEGO_ARM64_NEON) #include #include @@ -40,7 +42,10 @@ void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { uint32_t b1 = (bits16 >> 1) & 1u; uint32_t b2 = (bits16 >> 2) & 1u; uint32_t b3 = (bits16 >> 3) & 1u; - uint32x4_t bit_mask = {b0, b1, b2, b3}; + // Build the lane mask via vld1q_u32: MSVC's uint32x4_t is a union type, + // so GCC/Clang-style brace initialization of a vector is not portable. + const uint32_t bits[4] = {b0, b1, b2, b3}; + uint32x4_t bit_mask = vld1q_u32(bits); uint32x4_t sign_mask = vmulq_u32(bit_mask, sign_bit); float32x4_t v = vld1q_f32(&data[i]); v = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v), sign_mask)); @@ -104,4 +109,4 @@ void fht_inv_kacs_walk_neon(float *data, size_t len) { } // namespace ailego } // namespace zvec -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_ARM64_NEON diff --git a/src/ailego/math/fht_sse.cc b/src/ailego/math/fht_sse.cc index 1c03b1625..8aa4e743a 100644 --- a/src/ailego/math/fht_sse.cc +++ b/src/ailego/math/fht_sse.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #if defined(__SSE2__) #include diff --git a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc index 8b289b6e6..331818dc6 100644 --- a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc +++ b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc @@ -20,7 +20,7 @@ namespace ailego { //-------------------------------------------------- // Dense //-------------------------------------------------- -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) float InnerProductFp32NEON(const float *lhs, const float *rhs, size_t size); float MinusInnerProductFp32NEON(const float *lhs, const float *rhs, size_t size); @@ -49,7 +49,7 @@ float MinusInnerProductFp32Scalar(const float *lhs, const float *rhs, //! Compute the distance between matrix and query (FP32, M=1, N=1) void InnerProductMatrix::Compute(const float *m, const float *q, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) *out = InnerProductFp32NEON(m, q, dim); #else #if defined(__AVX512F__) @@ -73,14 +73,14 @@ void InnerProductMatrix::Compute(const float *m, const float *q, } #endif // __SSE__ *out = InnerProductFp32Scalar(m, q, dim); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } //! Compute the distance between matrix and query (FP32, M=1, N=1) void MinusInnerProductMatrix::Compute(const float *m, const float *q, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) *out = MinusInnerProductFp32NEON(m, q, dim); #else #if defined(__AVX512F__) @@ -104,7 +104,7 @@ void MinusInnerProductMatrix::Compute(const float *m, } #endif // __SSE__ *out = MinusInnerProductFp32Scalar(m, q, dim); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } //-------------------------------------------------- diff --git a/src/ailego/math/inner_product_matrix_fp32_neon.cc b/src/ailego/math/inner_product_matrix_fp32_neon.cc index c457b3ea2..fe344abf5 100644 --- a/src/ailego/math/inner_product_matrix_fp32_neon.cc +++ b/src/ailego/math/inner_product_matrix_fp32_neon.cc @@ -22,7 +22,7 @@ namespace ailego { //-------------------------------------------------- // Dense //-------------------------------------------------- -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) float InnerProductFp32NEON(const float *lhs, const float *rhs, size_t size) { const float *last = lhs + size; const float *last_aligned = lhs + ((size >> 3) << 3); @@ -59,7 +59,7 @@ float MinusInnerProductFp32NEON(const float *lhs, const float *rhs, return -1 * InnerProductFp32NEON(lhs, rhs, size); } -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc index b4f4c970d..dc31cad50 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_GNU_LIKE) #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) //! Compute the Inner Product between p and q, and each Squared L2-Norm value float InnerProductAndSquaredNormFp16NEON(const Float16 *lhs, const Float16 *rhs, @@ -153,7 +153,7 @@ float MipsEuclideanDistanceRepeatedQuadraticInjectionFp16NEON( return sum; } -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc index f48626a3f..1c54b1f57 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc @@ -18,7 +18,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, size_t size, float *sql, float *sqr); #endif @@ -63,7 +63,7 @@ float MipsInnerProductSparseInSegment(uint32_t m_sparse_count, //! Compute the distance between matrix and query by SphericalInjection void MipsSquaredEuclideanDistanceMatrix::Compute( const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { -#if __ARM_NEON +#if defined(AILEGO_HAVE_NEON) float u2{0.0f}; float v2{0.0f}; float sum = InnerProductAndSquaredNormFp32NEON(p, q, dim, &u2, &v2); @@ -91,14 +91,14 @@ void MipsSquaredEuclideanDistanceMatrix::Compute( #endif // __SSE__ *out = MipsEuclideanDistanceSphericalInjectionFp32Scalar(p, q, dim, e2); return; -#endif //__ARM_NEON +#endif // AILEGO_HAVE_NEON } //! Compute the distance between matrix and query by RepeatedQuadraticInjection void MipsSquaredEuclideanDistanceMatrix::Compute( const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) float u2{0.0f}; float v2{0.0f}; float sum = InnerProductAndSquaredNormFp32NEON(p, q, dim, &u2, &v2); @@ -140,7 +140,7 @@ void MipsSquaredEuclideanDistanceMatrix::Compute( e2); return; -#endif //__ARM_NEON +#endif // AILEGO_HAVE_NEON } // Sparse diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc index 6491f2260..c6eb428c3 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc @@ -19,7 +19,7 @@ namespace zvec { namespace ailego { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) //! Compute the Inner Product between p and q, and each Squared L2-Norm value float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, size_t size, float *sql, float *sqr) { @@ -71,7 +71,7 @@ float InnerProductAndSquaredNormFp32NEON(const float *lhs, const float *rhs, return result; } -#endif //__ARM_NEON +#endif // AILEGO_HAVE_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm1_matrix.h b/src/ailego/math/norm1_matrix.h index 7e8d9cbc8..fd42c389e 100644 --- a/src/ailego/math/norm1_matrix.h +++ b/src/ailego/math/norm1_matrix.h @@ -116,7 +116,7 @@ struct Norm1Matrix< } }; -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) /*! L1-Norm Matrix (FP32, M=1) */ template <> @@ -127,10 +127,11 @@ struct Norm1Matrix { //! Compute the L1-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || AILEGO_ARM64_NEON -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON +// specialization to gcc/clang aarch64. +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) /*! L1-Norm Matrix (FP16, M=1) */ template <> @@ -141,7 +142,7 @@ struct Norm1Matrix { //! Compute the L1-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm1_matrix_fp16.cc b/src/ailego/math/norm1_matrix_fp16.cc index e75b3e0a8..a05bc5a3f 100644 --- a/src/ailego/math/norm1_matrix_fp16.cc +++ b/src/ailego/math/norm1_matrix_fp16.cc @@ -67,12 +67,14 @@ static const __m512 ABS_MASK_FP32_AVX512 = //! Calculate sum of absolute (NEON) #define SA_FP16_NEON(v_m, v_sum) v_sum = vaddq_f16(vabsq_f16(v_m), v_sum); -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the NEON FP16 +// kernel is gated to gcc/clang aarch64. The generic Float16 path is used +// on MSVC ARM64. +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) //! Compute the L1-norm of vectors (FP16, M=1) void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NORM_FP16_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) @@ -84,7 +86,7 @@ void Norm1Matrix::Compute(const ValueType *m, size_t dim, NORM_FP16_1_AVX(m, dim, out, ) #endif } -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm1_matrix_fp32.cc b/src/ailego/math/norm1_matrix_fp32.cc index 2e7279118..e988d7411 100644 --- a/src/ailego/math/norm1_matrix_fp32.cc +++ b/src/ailego/math/norm1_matrix_fp32.cc @@ -56,11 +56,11 @@ namespace ailego { //! Calculate sum of absolute (NEON) #define SA_FP32_NEON(v_m, v_sum) v_sum = vaddq_f32(vabsq_f32(v_m), v_sum); -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) //! Compute the L1-norm of vectors (FP32, M=1) void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) NORM_FP32_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) @@ -78,7 +78,7 @@ void Norm1Matrix::Compute(const ValueType *m, size_t dim, NORM_FP32_1_SSE(m, dim, out, ) #endif } -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || AILEGO_ARM64_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm2_matrix.h b/src/ailego/math/norm2_matrix.h index 3c905147d..05d7890b4 100644 --- a/src/ailego/math/norm2_matrix.h +++ b/src/ailego/math/norm2_matrix.h @@ -371,7 +371,7 @@ struct SquaredNorm2Matrix= 2>::type> { } }; -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) /*! L2-Norm Matrix (FP32, M=1) */ template <> @@ -393,10 +393,11 @@ struct SquaredNorm2Matrix { //! Compute the squared L2-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || AILEGO_ARM64_NEON -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16; gate FP16 NEON +// specialization to gcc/clang aarch64. +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) /*! L2-Norm Matrix (FP16, M=1) */ template <> @@ -418,7 +419,7 @@ struct SquaredNorm2Matrix { //! Compute the squared L2-norm of vectors static void Compute(const ValueType *m, size_t dim, float *out); }; -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/norm2_matrix_fp16.cc b/src/ailego/math/norm2_matrix_fp16.cc index 6bb8dd06c..259d6e1e9 100644 --- a/src/ailego/math/norm2_matrix_fp16.cc +++ b/src/ailego/math/norm2_matrix_fp16.cc @@ -52,12 +52,13 @@ namespace ailego { //! Calculate sum of squared (NEON) #define SS_FP16_NEON(v_m, v_sum) v_sum = vfmaq_f16(v_sum, v_m, v_m); -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the NEON FP16 +// kernel is gated to gcc/clang aarch64. +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) //! Compute the L2-norm of vectors (FP16, M=1) void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NORM_FP16_1_NEON(m, dim, out, std::sqrt) #else #if defined(__AVX512F__) @@ -73,7 +74,7 @@ void Norm2Matrix::Compute(const ValueType *m, size_t dim, //! Compute the L2-norm of vectors (FP16, M=1) void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NORM_FP16_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) @@ -85,7 +86,7 @@ void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, NORM_FP16_1_AVX(m, dim, out, ) #endif } -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/norm2_matrix_fp32.cc b/src/ailego/math/norm2_matrix_fp32.cc index 8cc76c1f5..8124255f0 100644 --- a/src/ailego/math/norm2_matrix_fp32.cc +++ b/src/ailego/math/norm2_matrix_fp32.cc @@ -43,11 +43,11 @@ namespace ailego { //! Calculate sum of squared (NEON) #define SS_FP32_NEON(v_m, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_m); -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) //! Compute the L2-norm of vectors (FP32, M=1) void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) NORM_FP32_1_NEON(m, dim, out, std::sqrt) #else #if defined(__AVX512F__) @@ -69,7 +69,7 @@ void Norm2Matrix::Compute(const ValueType *m, size_t dim, //! Compute the squared L2-norm of vectors (FP32, M=1) void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) NORM_FP32_1_NEON(m, dim, out, ) #else #if defined(__AVX512F__) @@ -87,7 +87,7 @@ void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, NORM_FP32_1_SSE(m, dim, out, ) #endif } -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || AILEGO_ARM64_NEON } // namespace ailego } // namespace zvec diff --git a/src/ailego/math/normalizer.cc b/src/ailego/math/normalizer.cc index a31a9f350..239282e0f 100644 --- a/src/ailego/math/normalizer.cc +++ b/src/ailego/math/normalizer.cc @@ -17,7 +17,7 @@ namespace zvec { namespace ailego { -#if (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(AILEGO_ARM64_NEON) static inline void NormalizeNEON(float *arr, size_t dim, float norm) { float *last = arr + dim; float *last_aligned = arr + ((dim >> 3) << 3); @@ -43,6 +43,11 @@ static inline void NormalizeNEON(float *arr, size_t dim, float norm) { } } +// MSVC ARM64 does not declare `float16_t` unless ARMv8.2 FP16 is enabled +// (`_M_ARM64FP16` / `/arch:armv8.2`). Skip the FP16 NEON variants on MSVC +// ARM64 — the FP16 `Normalizer::Compute` dispatch below falls +// through to the scalar conversion path. +#if !defined(_MSC_VER) #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) static inline void NormalizeNEON(float16_t *arr, size_t dim, float norm) { float16_t *last = arr + dim; @@ -114,7 +119,8 @@ static inline void NormalizeNEON(float16_t *arr, size_t dim, float norm) { } } #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -#endif // __ARM_NEON && __aarch64__ +#endif // !_MSC_VER (FP16 NEON: gcc/clang aarch64 only) +#endif // AILEGO_ARM64_NEON #if defined(__AVX__) #if defined(__AVX512F__) @@ -392,10 +398,10 @@ static inline void NormalizeSSE(float *arr, size_t dim, float norm) { } #endif // __SSE__ -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) //! Compute the norm of vector void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) NormalizeNEON(arr, dim, norm); #else #if defined(__AVX512F__) @@ -411,15 +417,18 @@ void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { } #endif // __AVX__ NormalizeSSE(arr, dim, norm); -#endif // __ARM_NEON +#endif // AILEGO_HAVE_NEON } -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || AILEGO_ARM64_NEON -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 +// `NormalizeNEON` variants are not defined there (see top of file). +// Exclude MSVC from this specialization so MSVC ARM64 falls back to the +// generic uint16_t-based Float16 normalize path. +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) //! Compute the norm of vector void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { -#if defined(__ARM_NEON) +#if defined(AILEGO_ARM64_GNU_LIKE) NormalizeNEON(reinterpret_cast(arr), dim, norm); #else #if defined(__AVX512F__) @@ -429,9 +438,9 @@ void Normalizer::Compute(ValueType *arr, size_t dim, float norm) { } #endif // __AVX512F__ NormalizeAVX(reinterpret_cast(arr), dim, norm); -#endif // __ARM_NEON +#endif // AILEGO_ARM64_GNU_LIKE } -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/normalizer.h b/src/ailego/math/normalizer.h index 2c191b0e7..996a6a39f 100644 --- a/src/ailego/math/normalizer.h +++ b/src/ailego/math/normalizer.h @@ -51,7 +51,7 @@ struct Normalizer { } }; -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(__SSE__) || defined(AILEGO_ARM64_NEON) /*! Normalizer (FP32) */ template <> @@ -78,10 +78,11 @@ struct Normalizer { } } }; -#endif // __SSE__ || (__ARM_NEON && __aarch64__) +#endif // __SSE__ || AILEGO_ARM64_NEON -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) +// MSVC ARM64 lacks `float16_t` without ARMv8.2 FP16, so the FP16 NEON +// `Normalizer` specialization is gated to gcc/clang aarch64. +#if (defined(__F16C__) && defined(__AVX__)) || defined(AILEGO_ARM64_GNU_LIKE) /*! Normalizer (FP16) */ template <> @@ -108,7 +109,7 @@ struct Normalizer { } } }; -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) +#endif // (__F16C__ && __AVX__) || AILEGO_ARM64_GNU_LIKE } // namespace ailego } // namespace zvec diff --git a/src/ailego/utility/bitset_helper.cc b/src/ailego/utility/bitset_helper.cc index 19be34847..95912413d 100644 --- a/src/ailego/utility/bitset_helper.cc +++ b/src/ailego/utility/bitset_helper.cc @@ -23,7 +23,7 @@ #define bitset_popcount64 _mm_popcnt_u64 #endif // !__SSE4_2__ -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) static inline void bitset_and(uint32_t *lhs, const uint32_t *rhs, size_t size) { uint32_t *last = lhs + size; uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1480,7 +1480,7 @@ static inline bool bitset_test_none(const uint32_t *lhs, size_t size) { #endif // AILEGO_M64 #endif // __AVX2__ -#if (defined(__ARM_NEON) && defined(__aarch64__)) +#if defined(AILEGO_ARM64_NEON) static inline size_t bitset_cardinality(const uint32_t *lhs, size_t size) { const uint32_t *last = lhs + size; const uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1839,7 +1839,7 @@ static inline size_t bitset_or_cardinality(const uint32_t *lhs, return count; } -#else // !__ARM_NEON && !AILEGO_M64 +#else // !AILEGO_ARM64_NEON && !AILEGO_M64 static inline size_t bitset_cardinality(const uint32_t *lhs, size_t size) { const uint32_t *last = lhs + size; const uint32_t *last_aligned = lhs + ((size >> 2) << 2); @@ -1964,7 +1964,7 @@ static inline size_t bitset_or_cardinality(const uint32_t *lhs, } return count; } -#endif // __ARM_NEON && __aarch64__ +#endif // AILEGO_ARM64_NEON namespace zvec { diff --git a/src/ailego/utility/float_helper.cc b/src/ailego/utility/float_helper.cc index bc07eec25..78e5a700e 100644 --- a/src/ailego/utility/float_helper.cc +++ b/src/ailego/utility/float_helper.cc @@ -22,7 +22,9 @@ // #define float32(x) _cvtsh_ss(x) // #endif // __F16C__ && __AVX__ -#if defined(__aarch64__) +// `__fp16` is a GCC/Clang extension; MSVC ARM64 has no equivalent and +// falls through to the F16C/scalar paths below. +#if defined(AILEGO_ARM64_GNU_LIKE) static inline float float32(uint16_t val) { __fp16 f; memcpy(&f, &val, sizeof(val)); @@ -63,7 +65,7 @@ static inline void convert_fp32_to_fp16(const float *arr, size_t size, out[i] = float16(arr[i] / norm); } } -#else +#else // !AILEGO_ARM64_GNU_LIKE // Refer: https://github.com/Maratyszcza/FP16/blob/master/third-party/half.hpp static inline float float32(uint16_t val) { static const uint32_t mantissa_table[2048] = { @@ -1211,7 +1213,7 @@ static inline void convert_fp32_to_fp16(const float *arr, size_t size, return convert_fp32_to_fp16_fallback(arr, size, norm, out); } -#endif // +#endif // AILEGO_ARM64_GNU_LIKE namespace zvec { namespace ailego { diff --git a/src/ailego/version.i b/src/ailego/version.i index c1b14be2e..d6347cc3f 100644 --- a/src/ailego/version.i +++ b/src/ailego/version.i @@ -225,7 +225,7 @@ #define AILEGO_VERSION_OPENMP "" #endif -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) #define AILEGO_VERSION_SIMD " Arm Neon Instruction Set\n" #elif defined(__AVX512FP16__) #define AILEGO_VERSION_SIMD " AVX-512FP16 Instruction Set\n" diff --git a/src/include/zvec/ailego/buffer/concurrentqueue.h b/src/include/zvec/ailego/buffer/concurrentqueue.h index f7f3d77ed..1bff8b08e 100644 --- a/src/include/zvec/ailego/buffer/concurrentqueue.h +++ b/src/include/zvec/ailego/buffer/concurrentqueue.h @@ -129,8 +129,9 @@ static inline thread_id_t thread_id() { } } // namespace details } // namespace moodycamel -#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || \ - (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__) || \ +#elif defined(__arm__) || defined(_M_ARM) || \ + (defined(__aarch64__) || defined(_M_ARM64)) || \ + (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__) || \ defined(MOODYCAMEL_NO_THREAD_LOCAL) namespace moodycamel { namespace details { @@ -293,13 +294,14 @@ inline thread_id_t thread_id() { // support thread_local either. Finally, iOS/ARM doesn't have support for it // either, and g++/ARM allows it to compile but it's unconfirmed to actually // work -#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && \ - (!defined(__MINGW32__) && !defined(__MINGW64__) || \ - !defined(__WINPTHREADS_VERSION)) && \ - (!defined(__GNUC__) || __GNUC__ > 4 || \ - (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && \ - (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && \ - !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__) +#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && \ + (!defined(__MINGW32__) && !defined(__MINGW64__) || \ + !defined(__WINPTHREADS_VERSION)) && \ + (!defined(__GNUC__) || __GNUC__ > 4 || \ + (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && \ + (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && \ + !defined(_M_ARM) && !(defined(__aarch64__) || defined(_M_ARM64)) && \ + !defined(__MVS__) // Assume `thread_local` is fully supported in all other C++11 // compilers/platforms #define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // tentatively enabled for now; diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 2b1c1a0e1..b43b705d8 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -14,6 +14,34 @@ #pragma once +// Architecture / SIMD feature detection, named once so guards across the +// codebase read clearly instead of repeating compiler-macro disjunctions: +// +// AILEGO_ARM64 - AArch64. GCC/Clang: __aarch64__; MSVC: _M_ARM64. +// AILEGO_ARM - any ARM, 32- or 64-bit. +// AILEGO_HAVE_NEON - NEON intrinsics available. NEON is an ARMv8 +// baseline, so MSVC ARM64 qualifies. +// AILEGO_ARM64_NEON - AArch64 with NEON. Use for FP32 kernels. +// AILEGO_ARM64_GNU_LIKE - AArch64 NEON under GCC/Clang. Excludes MSVC, +// which exposes neither float16_t nor the v*_f16 +// intrinsics outside ARMv8.2 FP16. Use for FP16 +// kernels and other GCC/Clang-only extensions. +#if defined(__aarch64__) || defined(_M_ARM64) +#define AILEGO_ARM64 1 +#endif +#if defined(__arm__) || defined(AILEGO_ARM64) +#define AILEGO_ARM 1 +#endif +#if defined(__ARM_NEON) || defined(_M_ARM64) +#define AILEGO_HAVE_NEON 1 +#endif +#if defined(AILEGO_HAVE_NEON) && defined(AILEGO_ARM64) +#define AILEGO_ARM64_NEON 1 +#endif +#if defined(__ARM_NEON) && defined(__aarch64__) +#define AILEGO_ARM64_GNU_LIKE 1 +#endif + #if defined(_WIN32) || defined(_WIN64) #include #endif @@ -30,13 +58,16 @@ #if defined(_MSC_VER) #include +#if defined(_M_ARM64) +#include +#endif #else #include #include #if defined(__x86_64__) || defined(__i386) #include #endif -#if defined(__ARM_NEON) +#if defined(AILEGO_HAVE_NEON) #include #endif #if defined(__ARM_FEATURE_CRC32) @@ -111,7 +142,7 @@ extern "C" { #endif #if defined(__GNUC__) -#if defined(__x86_64__) || defined(__aarch64__) || defined(__ppc64__) +#if defined(__x86_64__) || defined(AILEGO_ARM64) || defined(__ppc64__) #define AILEGO_M64 #else #define AILEGO_M32 @@ -236,10 +267,12 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_popcount ailego_popcount32 #endif // AILEGO_M64 -#if defined(__arm__) || defined(__aarch64__) +#if defined(AILEGO_ARM) // ARMv7 Architecture Reference Manual (for YIELD) // ARM Compiler toolchain Compiler Reference (for __yield() instrinsic) -#if defined(__CC_ARM) +#if defined(__CC_ARM) || defined(_MSC_VER) +// ARM Compiler toolchain and MSVC both expose the intrinsic `__yield()` +// for the AArch64/ARMv7 YIELD instruction. #define ailego_yield() __yield() #else #define ailego_yield() __asm__ __volatile__("yield") @@ -248,7 +281,7 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_yield() _mm_pause() #else #define ailego_yield() ((void)0) -#endif // __arm__ || __aarch64__ +#endif // AILEGO_ARM #if defined(_MSC_VER) #define ailego_aligned_malloc(SIZE, ALIGN) \ @@ -281,11 +314,11 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 32) #elif defined(__SSE__) #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 16) -#elif defined(__ARM_NEON) +#elif defined(AILEGO_HAVE_NEON) #define ailego_malloc(SIZE) ailego_aligned_malloc((SIZE), 16) #endif #endif // !ailego_malloc -#if (defined(__SSE__) || defined(__ARM_NEON)) && !defined(ailego_free) +#if (defined(__SSE__) || defined(AILEGO_HAVE_NEON)) && !defined(ailego_free) #define ailego_free ailego_aligned_free #endif #endif // !__SANITIZE_ADDRESS__ diff --git a/src/include/zvec/ailego/utility/float_helper.h b/src/include/zvec/ailego/utility/float_helper.h index 575da1b07..9d04e38a4 100644 --- a/src/include/zvec/ailego/utility/float_helper.h +++ b/src/include/zvec/ailego/utility/float_helper.h @@ -16,6 +16,7 @@ #include #include +#include #include namespace zvec { @@ -53,7 +54,10 @@ struct ZVEC_AILEGO_API FloatHelper { } }; -#if !defined(__aarch64__) +// The `#else` branch below stores `Float16::value_` as `__fp16`, a GCC/Clang +// extension MSVC lacks even on ARM64. Keep the uint16_t storage path there so +// the wrapper still compiles. Must match the guard in float_helper.cc. +#if !defined(AILEGO_ARM64_GNU_LIKE) /*! Half-Precision Floating Point */ class Float16 { @@ -229,7 +233,7 @@ class Float16 { private: __fp16 value_; }; -#endif +#endif // !AILEGO_ARM64_GNU_LIKE // Check size of Float16 static_assert(sizeof(Float16) == 2, "Float16 must be aligned with 2 bytes"); diff --git a/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc b/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc index 24fa14304..5df1567eb 100644 --- a/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc +++ b/src/turbo/distance/neon/pq_quantizer_int8/pq_distance.cc @@ -18,15 +18,17 @@ // the running accumulator, halving the number of FP add operations. #include "neon/pq_quantizer_int8/pq_distance.h" -#if defined(__ARM_NEON) && defined(__aarch64__) +#include +#if defined(AILEGO_ARM64_NEON) #include #endif #include #include +#include "scalar/pq_quantizer_int8/pq_distance.h" namespace zvec::turbo::neon { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) namespace { // Horizontal sum of 4 floats in a float32x4_t register via pairwise add. @@ -43,7 +45,7 @@ inline float horizontal_sum_neon(float32x4_t v) { void pq_adc_int8_distance_neon(const void *pq_code_v, const void *lut_v, size_t num_chunk, float *out) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) constexpr int kNumCentroids = 256; constexpr int kChunkSize = 4; // NEON processes 4 floats at once const auto *pq_code = reinterpret_cast(pq_code_v); @@ -60,7 +62,8 @@ void pq_adc_int8_distance_neon(const void *pq_code_v, const void *lut_v, float d1 = lut[(m + 1) * kNumCentroids + pq_code[m + 1]]; float d2 = lut[(m + 2) * kNumCentroids + pq_code[m + 2]]; float d3 = lut[(m + 3) * kNumCentroids + pq_code[m + 3]]; - float32x4_t d = {d0, d1, d2, d3}; + const float lane[4] = {d0, d1, d2, d3}; + float32x4_t d = vld1q_f32(lane); acc = vaddq_f32(acc, d); } @@ -73,17 +76,18 @@ void pq_adc_int8_distance_neon(const void *pq_code_v, const void *lut_v, *out = sum; #else - (void)pq_code_v; - (void)lut_v; - (void)num_chunk; - (void)out; + // Without NEON this translation unit still compiles, so delegate to the + // scalar kernel. Never leave `out` unwritten: turbo.cc selects these entry + // points from CpuFeatures flags, and a no-op here would silently return + // whatever the caller's buffer already held. + scalar::pq_adc_int8_distance(pq_code_v, lut_v, num_chunk, out); #endif } void pq_sdc_int8_distance_neon(const void *a_v, const void *b_v, const void *dist_table_v, size_t num_chunk, float *out) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) constexpr int kNumCentroids = 256; constexpr int chunk = kNumCentroids * kNumCentroids; // 65536 constexpr int kChunkSize = 4; @@ -109,7 +113,8 @@ void pq_sdc_int8_distance_neon(const void *a_v, const void *b_v, float d3 = dist_table[(m + 3) * chunk + static_cast(a[m + 3]) * kNumCentroids + static_cast(b[m + 3])]; - float32x4_t d = {d0, d1, d2, d3}; + const float lane[4] = {d0, d1, d2, d3}; + float32x4_t d = vld1q_f32(lane); acc = vaddq_f32(acc, d); } @@ -124,18 +129,14 @@ void pq_sdc_int8_distance_neon(const void *a_v, const void *b_v, *out = sum; #else - (void)a_v; - (void)b_v; - (void)dist_table_v; - (void)num_chunk; - (void)out; + scalar::pq_sdc_int8_distance(a_v, b_v, dist_table_v, num_chunk, out); #endif } void pq_adc_int8_batch_distance_neon(const void **candidates_v, const void *lut_v, size_t num, size_t num_chunk, float *out) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) constexpr int kNumCentroids = 256; constexpr int kChunkSize = 4; constexpr int kBatch = 4; @@ -156,20 +157,26 @@ void pq_adc_int8_batch_distance_neon(const void **candidates_v, size_t m = 0; for (; m + kChunkSize <= num_chunk; m += kChunkSize) { - // Each chunk position has its own 256-entry LUT row. - const float *tab0 = lut + (m + 0) * kNumCentroids; - const float *tab1 = lut + (m + 1) * kNumCentroids; - const float *tab2 = lut + (m + 2) * kNumCentroids; - const float *tab3 = lut + (m + 3) * kNumCentroids; - - float32x4_t d0 = {tab0[c0[m + 0]], tab1[c0[m + 1]], tab2[c0[m + 2]], - tab3[c0[m + 3]]}; - float32x4_t d1 = {tab0[c1[m + 0]], tab1[c1[m + 1]], tab2[c1[m + 2]], - tab3[c1[m + 3]]}; - float32x4_t d2 = {tab0[c2[m + 0]], tab1[c2[m + 1]], tab2[c2[m + 2]], - tab3[c2[m + 3]]}; - float32x4_t d3 = {tab0[c3[m + 0]], tab1[c3[m + 1]], tab2[c3[m + 2]], - tab3[c3[m + 3]]}; + // Each lane holds a different chunk, so every lane needs its own + // sub-table: lane j reads chunk (m + j). Sharing one base here would + // score every lane against chunk m's centroids. + const float *t0 = lut + (m + 0) * kNumCentroids; + const float *t1 = lut + (m + 1) * kNumCentroids; + const float *t2 = lut + (m + 2) * kNumCentroids; + const float *t3 = lut + (m + 3) * kNumCentroids; + + const float lane0[4] = {t0[c0[m + 0]], t1[c0[m + 1]], t2[c0[m + 2]], + t3[c0[m + 3]]}; + const float lane1[4] = {t0[c1[m + 0]], t1[c1[m + 1]], t2[c1[m + 2]], + t3[c1[m + 3]]}; + const float lane2[4] = {t0[c2[m + 0]], t1[c2[m + 1]], t2[c2[m + 2]], + t3[c2[m + 3]]}; + const float lane3[4] = {t0[c3[m + 0]], t1[c3[m + 1]], t2[c3[m + 2]], + t3[c3[m + 3]]}; + float32x4_t d0 = vld1q_f32(lane0); + float32x4_t d1 = vld1q_f32(lane1); + float32x4_t d2 = vld1q_f32(lane2); + float32x4_t d3 = vld1q_f32(lane3); acc0 = vaddq_f32(acc0, d0); acc1 = vaddq_f32(acc1, d1); @@ -200,11 +207,7 @@ void pq_adc_int8_batch_distance_neon(const void **candidates_v, pq_adc_int8_distance_neon(candidates[i], lut, num_chunk, out + i); } #else - (void)candidates_v; - (void)lut_v; - (void)num; - (void)num_chunk; - (void)out; + scalar::pq_adc_int8_batch_distance(candidates_v, lut_v, num, num_chunk, out); #endif } diff --git a/src/turbo/distance/neon/rotate/fht/fht.cc b/src/turbo/distance/neon/rotate/fht/fht.cc index 0ffbfaad4..03419eb33 100644 --- a/src/turbo/distance/neon/rotate/fht/fht.cc +++ b/src/turbo/distance/neon/rotate/fht/fht.cc @@ -13,7 +13,8 @@ // limitations under the License. #include "fht.h" -#if defined(__ARM_NEON) && defined(__aarch64__) +#include +#if defined(AILEGO_ARM64_NEON) #include #endif #include @@ -26,7 +27,7 @@ namespace zvec::turbo::neon { void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) const uint32x4_t sign_bit = vdupq_n_u32(0x80000000u); size_t simd_end = dim & ~3u; size_t flip_bytes = (dim + 7) / 8; @@ -43,7 +44,10 @@ void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { uint32_t b1 = (bits16 >> 1) & 1u; uint32_t b2 = (bits16 >> 2) & 1u; uint32_t b3 = (bits16 >> 3) & 1u; - uint32x4_t bit_mask = {b0, b1, b2, b3}; + // Build the lane mask via vld1q_u32: MSVC's uint32x4_t is a union type, + // so GCC/Clang-style brace initialization of a vector is not portable. + const uint32_t bits[4] = {b0, b1, b2, b3}; + uint32x4_t bit_mask = vld1q_u32(bits); uint32x4_t sign_mask = vmulq_u32(bit_mask, sign_bit); float32x4_t v = vld1q_f32(&data[i]); v = vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(v), sign_mask)); @@ -56,14 +60,12 @@ void fht_flip_sign_neon(const uint8_t *flip, float *data, size_t dim) { } } #else - (void)flip; - (void)data; - (void)dim; + scalar::fht_flip_sign(flip, data, dim); #endif } void fht_kacs_walk_neon(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) size_t half = len / 2; size_t base = len % 2; size_t offset = base + half; @@ -85,13 +87,12 @@ void fht_kacs_walk_neon(float *data, size_t len) { data[half] *= std::sqrt(2.0f); } #else - (void)data; - (void)len; + scalar::fht_kacs_walk(data, len); #endif } void fht_inv_kacs_walk_neon(float *data, size_t len) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) size_t half = len / 2; size_t base = len % 2; size_t offset = base + half; @@ -114,13 +115,12 @@ void fht_inv_kacs_walk_neon(float *data, size_t len) { data[i + offset] = (a - b) * 0.5f; } #else - (void)data; - (void)len; + scalar::fht_inv_kacs_walk(data, len); #endif } void fht_vec_rescale_neon(float *data, size_t n, float factor) { -#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(AILEGO_ARM64_NEON) const float32x4_t fac = vdupq_n_f32(factor); size_t simd_end = n & ~3u; for (size_t i = 0; i < simd_end; i += 4) { @@ -132,39 +132,37 @@ void fht_vec_rescale_neon(float *data, size_t n, float factor) { data[i] *= factor; } #else - (void)data; - (void)n; - (void)factor; + scalar::fht_vec_rescale(data, n, factor); #endif } -void fht_rotate_neon(const float *in, float *out, size_t in_dim, - size_t /*out_dim*/, void *ctx) { -#if defined(__ARM_NEON) && defined(__aarch64__) +void fht_rotate_neon(const float *in, float *out, size_t in_dim, size_t out_dim, + void *ctx) { +#if defined(AILEGO_ARM64_NEON) + (void)out_dim; static constexpr FhtPrimitives kPrim = { fht_flip_sign_neon, scalar::fht_inplace, fht_kacs_walk_neon, fht_inv_kacs_walk_neon, fht_vec_rescale_neon}; fht_rotate_impl(in, out, in_dim, ctx, kPrim); #else - (void)in; - (void)out; - (void)in_dim; - (void)ctx; + // Without NEON this translation unit still compiles, so delegate to the + // scalar rotator. Never leave `out` unwritten: the runtime dispatcher in + // turbo.cc selects these entry points from CpuFeatures flags, and a no-op + // here would silently emit uninitialized vectors instead of rotated ones. + scalar::fht_rotate(in, out, in_dim, out_dim, ctx); #endif } void fht_unrotate_neon(const float *in, float *out, size_t in_dim, - size_t /*out_dim*/, void *ctx) { -#if defined(__ARM_NEON) && defined(__aarch64__) + size_t out_dim, void *ctx) { +#if defined(AILEGO_ARM64_NEON) + (void)out_dim; static constexpr FhtPrimitives kPrim = { fht_flip_sign_neon, scalar::fht_inplace, fht_kacs_walk_neon, fht_inv_kacs_walk_neon, fht_vec_rescale_neon}; fht_unrotate_impl(in, out, in_dim, ctx, kPrim); #else - (void)in; - (void)out; - (void)in_dim; - (void)ctx; + scalar::fht_unrotate(in, out, in_dim, out_dim, ctx); #endif } diff --git a/thirdparty/FastPFOR/CMakeLists.txt b/thirdparty/FastPFOR/CMakeLists.txt index 6ff7b8020..4fb95f2f2 100644 --- a/thirdparty/FastPFOR/CMakeLists.txt +++ b/thirdparty/FastPFOR/CMakeLists.txt @@ -52,3 +52,12 @@ cc_library( DEFS ${FASTPFOR_EXTRA_DEFS} CXXFLAGS ${FASTPFOR_EXTRA_CXXFLAGS} ) + +# Mark SIMDe's x86 emulation headers as a SYSTEM include so consumers of +# FastPFOR (e.g. src/db's bitpacked posting-list translation units) do not +# inherit SIMDe's warnings under strict /W4 /WX builds. Without this MSVC +# ARM64 fails on C4189 / C4127 / C4324 from simde/x86/mmx.h. +if(_FASTPFOR_NEEDS_SIMDE) + set_property(TARGET FastPFOR APPEND PROPERTY + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${simde_SOURCE_DIR}") +endif() diff --git a/thirdparty/arrow/CMakeLists.txt b/thirdparty/arrow/CMakeLists.txt index 6e01ac1f0..e497cc024 100644 --- a/thirdparty/arrow/CMakeLists.txt +++ b/thirdparty/arrow/CMakeLists.txt @@ -18,6 +18,10 @@ endif() if(MSVC) set(ARROW_WIN_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.windows.patch) apply_patch_once("arrow_windows_crt_fix" "${ARROW_SRC_DIR}" "${ARROW_WIN_PATCH}") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ARM64|arm64|aarch64)$") + set(ARROW_WIN_ARM64_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.windows-arm64.patch) + apply_patch_once("arrow_windows_arm64_fix" "${ARROW_SRC_DIR}" "${ARROW_WIN_ARM64_PATCH}") + endif() endif() # Trim unused compute kernels to reduce binary size (~14MB savings) set(ARROW_SLIM_COMPUTE_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.slim_compute.patch) @@ -145,6 +149,14 @@ elseif (MSVC) -DARROW_USE_STATIC_CRT=${ZVEC_USE_STATIC_CRT} "-DCMAKE_MSVC_RUNTIME_LIBRARY=${_ARROW_MSVC_RUNTIME}" ) + # Arrow 21.0's xsimd-13 does not provide make_sized_batch_t for MSVC ARM64, + # so disable SIMD on that target. x86/x64 MSVC keeps the default SSE4.2 path. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ARM64|arm64|aarch64)$") + list(APPEND ARROW_EXTRA_CMAKE_ARGS + -DARROW_SIMD_LEVEL=NONE + -DARROW_RUNTIME_SIMD_LEVEL=NONE + ) + endif() ExternalProject_Add( ARROW.BUILD PREFIX arrow SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 diff --git a/thirdparty/arrow/arrow.windows-arm64.patch b/thirdparty/arrow/arrow.windows-arm64.patch new file mode 100644 index 000000000..f815a0c49 --- /dev/null +++ b/thirdparty/arrow/arrow.windows-arm64.patch @@ -0,0 +1,35 @@ +diff --git a/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp b/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp +index 0181e69e4e..349e3b6bfa 100644 +--- a/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp ++++ b/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp +@@ -67,7 +67,8 @@ + #define PCG_LITTLE_ENDIAN 1 + #elif __BIG_ENDIAN__ || _BIG_ENDIAN + #define PCG_LITTLE_ENDIAN 0 +- #elif __x86_64 || __x86_64__ || _M_X64 || __i386 || __i386__ || _M_IX86 ++ #elif __x86_64 || __x86_64__ || _M_X64 || __i386 || __i386__ || _M_IX86 \ ++ || _M_ARM64 || _M_ARM || __aarch64__ || __arm__ + #define PCG_LITTLE_ENDIAN 1 + #elif __powerpc__ || __POWERPC__ || __ppc__ || __PPC__ \ + || __m68k__ || __mc68000__ +@@ -733,7 +734,7 @@ uint_x4 operator*(const uint_x4& a, + } + + #if PCG_64BIT_SPECIALIZATIONS +-#if defined(_MSC_VER) ++#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_ARM) + #pragma intrinsic(_umul128) + #endif + +@@ -742,7 +743,10 @@ template + uint_x4 operator*(const uint_x4& a, + const uint_x4& b) + { +-#if defined(_MSC_VER) ++#if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM)) ++ uint64_t lo = a.d.v01 * b.d.v01; ++ uint64_t hi = __umulh(a.d.v01, b.d.v01); ++#elif defined(_MSC_VER) + uint64_t hi; + uint64_t lo = _umul128(a.d.v01, b.d.v01, &hi); + #else