ext/bcmath: In the arm processor environment, NEON is used to use SIMD. (#18130)

Author: SakiTakamachi

ext/bcmath/libbcmath/src/convert.c

@@ -17,24 +17,22 @@
 #include "bcmath.h"
 #include "convert.h"
 #include "private.h"
-#ifdef __SSE2__
-# include <emmintrin.h>
-#endif
+#include "simd.h"
 
 char *bc_copy_and_toggle_bcd(char *restrict dest, const char *source, const char *source_end)
 {
 	const size_t bulk_shift = SWAR_REPEAT('0');
 
-#ifdef __SSE2__
-	/* SIMD SSE2 bulk shift + copy */
-	__m128i shift_vector = _mm_set1_epi8('0');
-	while (source + sizeof(__m128i) <= source_end) {
-		__m128i bytes = _mm_loadu_si128((const __m128i *) source);
-		bytes = _mm_xor_si128(bytes, shift_vector);
-		_mm_storeu_si128((__m128i *) dest, bytes);
+#ifdef HAVE_BC_SIMD_128
+	/* SIMD SSE2 or NEON bulk shift + copy */
+	bc_simd_128_t shift_vector = bc_simd_set_8x16('0');
+	while (source + sizeof(bc_simd_128_t) <= source_end) {
+		bc_simd_128_t bytes = bc_simd_load_8x16((const bc_simd_128_t *) source);
+		bytes = bc_simd_xor_8x16(bytes, shift_vector);
+		bc_simd_store_8x16((bc_simd_128_t *) dest, bytes);
 
-		source += sizeof(__m128i);
-		dest += sizeof(__m128i);
+		source += sizeof(bc_simd_128_t);
+		dest += sizeof(bc_simd_128_t);
 	}
 #endif
 

ext/bcmath/libbcmath/src/simd.h

@@ -0,0 +1,59 @@
+/*
+   +----------------------------------------------------------------------+
+   | Copyright (c) The PHP Group                                          |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | https://www.php.net/license/3_01.txt                                 |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+   | Authors: Saki Takamachi <saki@php.net>                               |
+   +----------------------------------------------------------------------+
+*/
+
+
+#ifndef _BCMATH_SIMD_H_
+#define _BCMATH_SIMD_H_
+
+#ifdef __SSE2__
+# include <emmintrin.h>
+  typedef __m128i bc_simd_128_t;
+# define HAVE_BC_SIMD_128
+# define bc_simd_set_8x16(x) _mm_set1_epi8(x)
+# define bc_simd_load_8x16(ptr) _mm_loadu_si128((const __m128i *) (ptr))
+# define bc_simd_xor_8x16(a, b) _mm_xor_si128(a, b)
+# define bc_simd_store_8x16(ptr, val) _mm_storeu_si128((__m128i *) (ptr), val)
+# define bc_simd_add_8x16(a, b) _mm_add_epi8(a, b)
+# define bc_simd_cmpeq_8x16(a, b) _mm_cmpeq_epi8(a, b)
+# define bc_simd_cmplt_8x16(a, b) _mm_cmplt_epi8(a, b)
+# define bc_simd_movemask_8x16(a) _mm_movemask_epi8(a)
+
+#elif defined(__aarch64__) || defined(_M_ARM64)
+# include <arm_neon.h>
+  typedef int8x16_t bc_simd_128_t;
+# define HAVE_BC_SIMD_128
+# define bc_simd_set_8x16(x) vdupq_n_s8(x)
+# define bc_simd_load_8x16(ptr) vld1q_s8((const int8_t *) (ptr))
+# define bc_simd_xor_8x16(a, b) veorq_s8(a, b)
+# define bc_simd_store_8x16(ptr, val) vst1q_s8((int8_t *) (ptr), val)
+# define bc_simd_add_8x16(a, b) vaddq_s8(a, b)
+# define bc_simd_cmpeq_8x16(a, b) (vreinterpretq_s8_u8(vceqq_s8(a, b)))
+# define bc_simd_cmplt_8x16(a, b) (vreinterpretq_s8_u8(vcltq_s8(a, b)))
+  static inline int bc_simd_movemask_8x16(int8x16_t vec)
+  {
+      /**
+       * based on code from
+       * https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
+       */
+      uint16x8_t high_bits = vreinterpretq_u16_u8(vshrq_n_u8(vreinterpretq_u8_s8(vec), 7));
+      uint32x4_t paired16 = vreinterpretq_u32_u16(vsraq_n_u16(high_bits, high_bits, 7));
+      uint64x2_t paired32 = vreinterpretq_u64_u32(vsraq_n_u32(paired16, paired16, 14));
+      uint8x16_t paired64 = vreinterpretq_u8_u64(vsraq_n_u64(paired32, paired32, 28));
+      return vgetq_lane_u8(paired64, 0) | ((int) vgetq_lane_u8(paired64, 8) << 8);
+  }
+#endif
+
+#endif

ext/bcmath/libbcmath/src/str2num.c

@@ -32,30 +32,28 @@
 #include "bcmath.h"
 #include "convert.h"
 #include "private.h"
+#include "simd.h"
 #include <stdbool.h>
 #include <stddef.h>
-#ifdef __SSE2__
-# include <emmintrin.h>
-#endif
 
 /* Convert strings to bc numbers.  Base 10 only.*/
-static const char *bc_count_digits(const char *str, const char *end)
+static inline const char *bc_count_digits(const char *str, const char *end)
 {
 	/* Process in bulk */
-#ifdef __SSE2__
-	const __m128i offset = _mm_set1_epi8((signed char) (SCHAR_MIN - '0'));
+#ifdef HAVE_BC_SIMD_128
+	const bc_simd_128_t offset = bc_simd_set_8x16((signed char) (SCHAR_MIN - '0'));
 	/* we use the less than comparator, so add 1 */
-	const __m128i threshold = _mm_set1_epi8(SCHAR_MIN + ('9' + 1 - '0'));
+	const bc_simd_128_t threshold = bc_simd_set_8x16(SCHAR_MIN + ('9' + 1 - '0'));
 
-	while (str + sizeof(__m128i) <= end) {
-		__m128i bytes = _mm_loadu_si128((const __m128i *) str);
+	while (str + sizeof(bc_simd_128_t) <= end) {
+		bc_simd_128_t bytes = bc_simd_load_8x16((const bc_simd_128_t *) str);
 		/* Wrapping-add the offset to the bytes, such that all bytes below '0' are positive and others are negative.
 		 * More specifically, '0' will be -128 and '9' will be -119. */
-		bytes = _mm_add_epi8(bytes, offset);
+		bytes = bc_simd_add_8x16(bytes, offset);
 		/* Now mark all bytes that are <= '9', i.e. <= -119, i.e. < -118, i.e. the threshold. */
-		bytes = _mm_cmplt_epi8(bytes, threshold);
+		bytes = bc_simd_cmplt_8x16(bytes, threshold);
 
-		int mask = _mm_movemask_epi8(bytes);
+		int mask = bc_simd_movemask_8x16(bytes);
 		if (mask != 0xffff) {
 			/* At least one of the bytes is not within range. Move to the first offending byte. */
 #ifdef PHP_HAVE_BUILTIN_CTZL
@@ -65,7 +63,7 @@ static const char *bc_count_digits(const char *str, const char *end)
 #endif
 		}
 
-		str += sizeof(__m128i);
+		str += sizeof(bc_simd_128_t);
 	}
 #endif
 
@@ -79,19 +77,19 @@ static const char *bc_count_digits(const char *str, const char *end)
 static inline const char *bc_skip_zero_reverse(const char *scanner, const char *stop)
 {
 	/* Check in bulk */
-#ifdef __SSE2__
-	const __m128i c_zero_repeat = _mm_set1_epi8('0');
-	while (scanner - sizeof(__m128i) >= stop) {
-		scanner -= sizeof(__m128i);
-		__m128i bytes = _mm_loadu_si128((const __m128i *) scanner);
+#ifdef HAVE_BC_SIMD_128
+	const bc_simd_128_t c_zero_repeat = bc_simd_set_8x16('0');
+	while (scanner - sizeof(bc_simd_128_t) >= stop) {
+		scanner -= sizeof(bc_simd_128_t);
+		bc_simd_128_t bytes = bc_simd_load_8x16((const bc_simd_128_t *) scanner);
 		/* Checks if all numeric strings are equal to '0'. */
-		bytes = _mm_cmpeq_epi8(bytes, c_zero_repeat);
+		bytes = bc_simd_cmpeq_8x16(bytes, c_zero_repeat);
 
-		int mask = _mm_movemask_epi8(bytes);
+		int mask = bc_simd_movemask_8x16(bytes);
 		/* The probability of having 16 trailing 0s in a row is very low, so we use EXPECTED. */
 		if (EXPECTED(mask != 0xffff)) {
 			/* Move the pointer back and check each character in loop. */
-			scanner += sizeof(__m128i);
+			scanner += sizeof(bc_simd_128_t);
 			break;
 		}
 	}