From a1e60532eccded6a485ef2cff60d4f1ceac01a3a Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathandbossart@gmail.com>
Date: Fri, 26 Aug 2022 10:47:35 -0700
Subject: [PATCH v4 2/2] use ARM Advanced SIMD intrinsic functions where
 available

---
 src/include/port/simd.h | 46 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 43 insertions(+), 3 deletions(-)

diff --git a/src/include/port/simd.h b/src/include/port/simd.h
index c42dccf784..127b1f3f70 100644
--- a/src/include/port/simd.h
+++ b/src/include/port/simd.h
@@ -33,6 +33,19 @@
 typedef __m128i Vector8;
 typedef __m128i Vector32;
 
+#elif defined(__aarch64__)
+/*
+ * Include arm_neon.h if the compiler is targeting an architecture that
+ * supports ARM Advanced SIMD (Neon) intrinsics.  While Neon support is
+ * technically optional for aarch64, we assume it's unlikely that anyone will
+ * run PostgreSQL on specialized hardware lacking this feature, and we assume
+ * that compilers targeting this architecture understand Neon intrinsics.
+ */
+#include <arm_neon.h>
+#define USE_NEON
+typedef uint8x16_t Vector8;
+typedef uint32x4_t Vector32;
+
 #else
 /*
  * If no SIMD instructions are available, we can in some cases emulate vector
@@ -82,6 +95,8 @@ vector8_load(Vector8 *v, const uint8 *s)
 {
 #if defined(USE_SSE2)
 	*v = _mm_loadu_si128((const __m128i *) s);
+#elif defined(USE_NEON)
+	*v = vld1q_u8(s);
 #else
 	memcpy(v, s, sizeof(Vector8));
 #endif
@@ -92,6 +107,8 @@ vector32_load(Vector32 *v, const uint32 *s)
 {
 #ifdef USE_SSE2
 	*v = _mm_loadu_si128((const __m128i *) s);
+#elif defined(USE_NEON)
+	*v = vld1q_u32(s);
 #else
 	memcpy(v, s, sizeof(Vector32));
 #endif
@@ -106,6 +123,8 @@ vector8_broadcast(const uint8 c)
 {
 #if defined(USE_SSE2)
 	return _mm_set1_epi8(c);
+#elif defined(USE_NEON)
+	return vdupq_n_u8(c);
 #else
 	return ~UINT64CONST(0) / 0xFF * c;
 #endif
@@ -116,6 +135,8 @@ vector32_broadcast(const uint32 c)
 {
 #ifdef USE_SSE2
 	return _mm_set1_epi32(c);
+#elif defined(USE_NEON)
+	return vdupq_n_u32(c);
 #else
 	return ~UINT64CONST(0) / 0xFFFFFFFF * c;
 #endif
@@ -148,6 +169,8 @@ vector8_has(const Vector8 v, const uint8 c)
 	result = vector8_has_zero(v ^ vector8_broadcast(c));
 #elif defined(USE_SSE2)
 	result = _mm_movemask_epi8(_mm_cmpeq_epi8(v, vector8_broadcast(c)));
+#elif defined(USE_NEON)
+	result = vmaxvq_u8(vceqq_u8(v, vector8_broadcast(c))) != 0;
 #endif
 
 	Assert(assert_result == result);
@@ -166,7 +189,7 @@ vector8_has_zero(const Vector8 v)
 	 * definition.
 	 */
 	return vector8_has_le(v, 0);
-#elif defined(USE_SSE2)
+#elif defined(USE_SSE2) || defined(USE_NEON)
 	return vector8_has(v, 0);
 #endif
 }
@@ -179,8 +202,8 @@ static inline bool
 vector8_has_le(const Vector8 v, const uint8 c)
 {
 	bool		result = false;
-#if defined(USE_SSE2)
-	__m128i		sub;
+#ifndef USE_NO_SIMD
+	Vector8		sub;
 #endif
 
 	/* pre-compute the result for assert checking */
@@ -227,6 +250,11 @@ vector8_has_le(const Vector8 v, const uint8 c)
 	 */
 	sub = _mm_subs_epu8(v, vector8_broadcast(c));
 	result = vector8_has_zero(sub);
+#elif defined(USE_NEON)
+
+	/* use the same approach as the USE_SSE2 block above */
+	sub = vqsubq_u8(v, vector8_broadcast(c));
+	result = vector8_has_zero(sub);
 #endif
 
 	Assert(assert_result == result);
@@ -241,6 +269,8 @@ vector8_is_highbit_set(const Vector8 v)
 {
 #ifdef USE_SSE2
 	return _mm_movemask_epi8(v) != 0;
+#elif defined(USE_NEON)
+	return vmaxvq_u8(vandq_u8(v, vector8_broadcast(0x80))) != 0;
 #else
 	return v & vector8_broadcast(0x80);
 #endif
@@ -254,6 +284,8 @@ vector8_or(const Vector8 v1, const Vector8 v2)
 {
 #ifdef USE_SSE2
 	return _mm_or_si128(v1, v2);
+#elif defined(USE_NEON)
+	return vorrq_u8(v1, v2);
 #else
 	return v1 | v2;
 #endif
@@ -264,6 +296,8 @@ vector32_or(const Vector32 v1, const Vector32 v2)
 {
 #ifdef USE_SSE2
 	return _mm_or_si128(v1, v2);
+#elif defined(USE_NEON)
+	return vorrq_u32(v1, v2);
 #else
 	return v1 | v2;
 #endif
@@ -282,6 +316,8 @@ vector8_eq(const Vector8 v1, const Vector8 v2)
 {
 #ifdef USE_SSE2
 	return _mm_cmpeq_epi8(v1, v2);
+#elif defined(USE_NEON)
+	return vceqq_u8(v1, v2);
 #endif
 }
 
@@ -290,6 +326,8 @@ vector32_eq(const Vector32 v1, const Vector32 v2)
 {
 #ifdef USE_SSE2
 	return _mm_cmpeq_epi32(v1, v2);
+#elif defined(USE_NEON)
+	return vceqq_u32(v1, v2);
 #endif
 }
 
@@ -306,6 +344,8 @@ vector32_any_lane_set(const Vector32 v)
 {
 #ifdef USE_SSE2
 	return _mm_movemask_epi8(v) != 0;
+#elif defined(USE_NEON)
+	return vmaxvq_u32(v) != 0;
 #endif
 }
 
-- 
2.25.1