From ca3d7a87745e9743877a522f9920fe2335a98fec Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 23 Mar 2026 12:31:58 +0000 Subject: [PATCH v3 3/3] Add RISC-V CRC32C using the Zbc extension This adds hardware-accelerated CRC-32C computation for RISC-V platforms with the Zbc (carry-less multiply) or Zbkc (crypto carry-less) extension. The implementation uses the clmul and clmulh instructions for polynomial folding with Barrett reduction to compute CRC-32C checksums. This provides approximately 20x speedup over the software slicing-by-8 implementation. The algorithm is based on the Google Abseil project's RISC-V CRC32C implementation (https://github.com/abseil/abseil-cpp/pull/1986 in absl/crc/internal/crc_riscv.cc) that is Copyright 2025 The Abseil Authors licensed under the Apache License, Version 2.0. Runtime detection uses the Linux riscv_hwprobe syscall (kernel 6.4+) to check for Zbc/Zbkc support, falling back gracefully to software on older kernels or non-Linux platforms. Similar to ARMv8 CRC Extension and x86 SSE 4.2 support, this is compiled with '-march=rv64gc_zbc' and selected at runtime based on CPU capabilities. --- config/c-compiler.m4 | 41 +++++ configure.ac | 36 ++++- meson.build | 36 +++++ src/include/port/pg_crc32c.h | 14 ++ src/port/meson.build | 3 + src/port/pg_crc32c_riscv_choose.c | 101 ++++++++++++ src/port/pg_crc32c_riscv_zbc.c | 257 ++++++++++++++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 8 files changed, 482 insertions(+), 7 deletions(-) create mode 100644 src/port/pg_crc32c_riscv_choose.c create mode 100644 src/port/pg_crc32c_riscv_zbc.c diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index 629572ee350..91cc0688808 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -791,6 +791,47 @@ fi undefine([Ac_cachevar])dnl ])# PGAC_LOONGARCH_CRC32C_INTRINSICS +# PGAC_RISCV_ZBC_CRC32C_INTRINSICS +# --------------------------------- +# Check if the compiler supports RISC-V Zbc (carry-less multiply) instructions +# for CRC-32C computation, using inline assembly for clmul instruction. +# +# An optional compiler flag can be passed as argument (e.g. -march=rv64gc_zbc). +# If the intrinsics are supported, sets pgac_riscv_zbc_crc32c_intrinsics and +# CFLAGS_CRC. +# +# The Zbc extension provides clmul and clmulh instructions which are used with +# polynomial folding to compute CRC-32C. This implementation is based on the +# algorithm from Google Abseil (https://github.com/abseil/abseil-cpp/pull/1986). +AC_DEFUN([PGAC_RISCV_ZBC_CRC32C_INTRINSICS], +[define([Ac_cachevar], [AS_TR_SH([pgac_cv_riscv_zbc_crc32c_intrinsics_$1])])dnl +AC_CACHE_CHECK([for RISC-V Zbc clmul with CFLAGS=$1], [Ac_cachevar], +[pgac_save_CFLAGS=$CFLAGS +CFLAGS="$pgac_save_CFLAGS $1" +AC_LINK_IFELSE([AC_LANG_PROGRAM([ +#if !defined(__riscv) || !defined(__riscv_xlen) || __riscv_xlen != 64 +#error not RISC-V 64-bit +#endif + +static inline unsigned long clmul_test(unsigned long a, unsigned long b) +{ + unsigned long result; + __asm__("clmul %0, %1, %2" : "=r"(result) : "r"(a), "r"(b)); + return result; +}], + [unsigned long result = clmul_test(0x123, 0x456); + /* return computed value, to prevent the above being optimized away */ + return result == 0;])], + [Ac_cachevar=yes], + [Ac_cachevar=no]) +CFLAGS="$pgac_save_CFLAGS"]) +if test x"$Ac_cachevar" = x"yes"; then + CFLAGS_CRC="$1" + pgac_riscv_zbc_crc32c_intrinsics=yes +fi +undefine([Ac_cachevar])dnl +])# PGAC_RISCV_ZBC_CRC32C_INTRINSICS + # PGAC_XSAVE_INTRINSICS # --------------------- # Check if the compiler supports the XSAVE instructions using the _xgetbv diff --git a/configure.ac b/configure.ac index 8c5970d1be0..1f8a4fa51d4 100644 --- a/configure.ac +++ b/configure.ac @@ -2215,6 +2215,17 @@ fi # with the default compiler flags. PGAC_LOONGARCH_CRC32C_INTRINSICS() +# Check for RISC-V Zbc (carry-less multiply) for CRC calculations. +# +# The Zbc extension provides clmul and clmulh instructions for hardware- +# accelerated CRC-32C computation using polynomial folding. Check if we +# can compile with -march=rv64gc_zbc flag. CFLAGS_CRC is set if the flag +# is required. +# +# This implementation is based on Google Abseil's algorithm: +# https://github.com/abseil/abseil-cpp/pull/1986 +PGAC_RISCV_ZBC_CRC32C_INTRINSICS([-march=rv64gc_zbc]) + AC_SUBST(CFLAGS_CRC) # Select CRC-32C implementation. @@ -2245,7 +2256,7 @@ AC_SUBST(CFLAGS_CRC) # # If we are targeting a LoongArch processor, CRC instructions are # always available (at least on 64 bit), so no runtime check is needed. -if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then +if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x"" && test x"$USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK" = x""; then # Use Intel SSE 4.2 if available. if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then USE_SSE42_CRC32C=1 @@ -2267,9 +2278,14 @@ if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then USE_LOONGARCH_CRC32C=1 else - # fall back to slicing-by-8 algorithm, which doesn't require any - # special CPU support. - USE_SLICING_BY_8_CRC32C=1 + # RISC-V Zbc CRC, with runtime check. + if test x"$pgac_riscv_zbc_crc32c_intrinsics" = x"yes"; then + USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK=1 + else + # fall back to slicing-by-8 algorithm, which doesn't require any + # special CPU support. + USE_SLICING_BY_8_CRC32C=1 + fi fi fi fi @@ -2304,9 +2320,15 @@ else PG_CRC32C_OBJS="pg_crc32c_loongarch.o" AC_MSG_RESULT(LoongArch CRCC instructions) else - AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).]) - PG_CRC32C_OBJS="pg_crc32c_sb8.o" - AC_MSG_RESULT(slicing-by-8) + if test x"$USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then + AC_DEFINE(USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use RISC-V Zbc CRC instructions with a runtime check.]) + PG_CRC32C_OBJS="pg_crc32c_riscv_zbc.o pg_crc32c_sb8.o pg_crc32c_riscv_choose.o" + AC_MSG_RESULT(RISC-V Zbc instructions with runtime check) + else + AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).]) + PG_CRC32C_OBJS="pg_crc32c_sb8.o" + AC_MSG_RESULT(slicing-by-8) + fi fi fi fi diff --git a/meson.build b/meson.build index dac6fbdeb1e..3fbf8756ab7 100644 --- a/meson.build +++ b/meson.build @@ -2733,6 +2733,42 @@ int main(void) have_optimized_crc = true endif +elif host_cpu == 'riscv64' + + # Check for RISC-V Zbc (carry-less multiply) extension for CRC-32C. + # The Zbc extension provides clmul and clmulh instructions used for + # hardware-accelerated CRC computation via polynomial folding. + # + # This implementation is based on Google Abseil's algorithm: + # https://github.com/abseil/abseil-cpp/pull/1986 + + prog = ''' +#if !defined(__riscv) || !defined(__riscv_xlen) || __riscv_xlen != 64 +#error not RISC-V 64-bit +#endif + +static inline unsigned long clmul(unsigned long a, unsigned long b) +{ + unsigned long result; + __asm__("clmul %0, %1, %2" : "=r"(result) : "r"(a), "r"(b)); + return result; +} + +int main(void) +{ + unsigned long result = clmul(0x123, 0x456); + return result == 0; +} +''' + + if cc.links(prog, name: 'RISC-V Zbc clmul with -march=rv64gc_zbc', + args: test_c_args + ['-march=rv64gc_zbc']) + # Use RISC-V Zbc CRC, with runtime check + cflags_crc += '-march=rv64gc_zbc' + cdata.set('USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK', 1) + have_optimized_crc = true + endif + endif if not have_optimized_crc diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h index 9ac619aec3e..27fa27f773e 100644 --- a/src/include/port/pg_crc32c.h +++ b/src/include/port/pg_crc32c.h @@ -142,6 +142,20 @@ extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len) extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); +#elif defined(USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK) + +/* + * Use RISC-V Zbc instructions, but perform a runtime check first + * to check that they are available. + */ +#define COMP_CRC32C(crc, data, len) \ + ((crc) = pg_comp_crc32c((crc), (data), (len))) +#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + +extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +extern pg_crc32c pg_comp_crc32c_riscv_zbc(pg_crc32c crc, const void *data, size_t len); + #else /* * Use slicing-by-8 algorithm. diff --git a/src/port/meson.build b/src/port/meson.build index 9d0bb59aca0..e7c1bccf9d5 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -99,6 +99,9 @@ replace_funcs_pos = [ ['pg_crc32c_loongarch', 'USE_LOONGARCH_CRC32C'], # riscv + ['pg_crc32c_riscv_zbc', 'USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK', 'crc'], + ['pg_crc32c_riscv_choose', 'USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK'], + ['pg_crc32c_sb8', 'USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK'], ['pg_popcount_riscv', 'USE_RISCV_ZBB_WITH_RUNTIME_CHECK', 'zbb'], # generic fallback diff --git a/src/port/pg_crc32c_riscv_choose.c b/src/port/pg_crc32c_riscv_choose.c new file mode 100644 index 00000000000..18d105e5e12 --- /dev/null +++ b/src/port/pg_crc32c_riscv_choose.c @@ -0,0 +1,101 @@ +/*------------------------------------------------------------------------- + * + * pg_crc32c_riscv_choose.c + * Choose between RISC-V Zbc and software CRC-32C implementation. + * + * On first call, checks if the CPU supports the RISC-V Zbc (or Zbkc) extension. + * If it does, use carry-less multiply instructions for CRC-32C computation. + * Otherwise, fall back to the pure software implementation (slicing-by-8). + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/port/pg_crc32c_riscv_choose.c + * + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include +#include + +#include "port/pg_crc32c.h" + +/* + * RISC-V hardware probing definitions + */ +#ifndef __NR_riscv_hwprobe +#define __NR_riscv_hwprobe 258 +#endif + +#ifndef RISCV_HWPROBE_KEY_IMA_EXT_0 +#define RISCV_HWPROBE_KEY_IMA_EXT_0 4 +#endif + +#ifndef RISCV_HWPROBE_EXT_ZBC +#define RISCV_HWPROBE_EXT_ZBC (1ULL << 7) +#endif + +#ifndef RISCV_HWPROBE_EXT_ZBKC +#define RISCV_HWPROBE_EXT_ZBKC (1ULL << 27) +#endif + +struct riscv_hwprobe +{ + int64 key; + uint64 value; +}; + +/* + * Check if RISC-V Zbc or Zbkc extension is available + * + * Uses the riscv_hwprobe syscall which is available on Linux kernel 6.4+ + * Falls back to software if the syscall fails or extensions are not available. + */ +static bool +pg_crc32c_riscv_zbc_available(void) +{ +#if defined(__linux__) && defined(__riscv) && (__riscv_xlen == 64) + struct riscv_hwprobe pair = {.key = RISCV_HWPROBE_KEY_IMA_EXT_0}; + + /* + * Make the syscall. If it fails (e.g., old kernel, non-Linux), fall back + * to software. + */ + if (syscall(__NR_riscv_hwprobe, &pair, 1, 0, NULL, 0) != 0) + return false; + + /* + * Check if either Zbc (general bitmanip carry-less) or Zbkc (crypto + * carry-less) is available. Both provide clmul/clmulh instructions. + */ + return (pair.value & (RISCV_HWPROBE_EXT_ZBC | RISCV_HWPROBE_EXT_ZBKC)) != 0; +#else + /* Not on RISC-V Linux, or not 64-bit - use software fallback */ + return false; +#endif +} + +/* + * This gets called on the first call. It replaces the function pointer + * so that subsequent calls are routed directly to the chosen implementation. + */ +static pg_crc32c +pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len) +{ + if (pg_crc32c_riscv_zbc_available()) + pg_comp_crc32c = pg_comp_crc32c_riscv_zbc; + else + pg_comp_crc32c = pg_comp_crc32c_sb8; + + return pg_comp_crc32c(crc, data, len); +} + +pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len) = pg_comp_crc32c_choose; diff --git a/src/port/pg_crc32c_riscv_zbc.c b/src/port/pg_crc32c_riscv_zbc.c new file mode 100644 index 00000000000..9eb845dca69 --- /dev/null +++ b/src/port/pg_crc32c_riscv_zbc.c @@ -0,0 +1,257 @@ +/*------------------------------------------------------------------------- + * + * pg_crc32c_riscv_zbc.c + * Compute CRC-32C checksum using RISC-V Zbc carry-less multiply instructions + * + * This implementation uses the RISC-V Zbc (or Zbkc) extension for hardware- + * accelerated CRC-32C computation. It uses carry-less multiplication (clmul + * and clmulh) with polynomial folding and Barrett reduction. + * + * The algorithm is based on Google Abseil's implementation: + * https://github.com/abseil/abseil-cpp/pull/1986 + * File: absl/crc/internal/crc_riscv.cc + * + * Copyright 2025 The Abseil Authors + * Licensed under the Apache License, Version 2.0 + * Adapted for PostgreSQL under PostgreSQL license + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/port/pg_crc32c_riscv_zbc.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" + +#ifdef WORDS_BIGENDIAN +#error "RISC-V Zbc CRC implementation does not support big-endian systems" +#endif + +#include "port/pg_crc32c.h" + +/* + * 128-bit value for polynomial arithmetic + */ +typedef struct +{ + uint64 lo; + uint64 hi; +} V128; + +/* + * Carry-less multiply instructions from RISC-V Zbc/Zbkc extension + */ +static inline uint64 +pg_clmul(uint64 a, uint64 b) +{ + uint64 _res; + + __asm__( + " clmul %0, %1, %2\n" +: "=r"(_res) +: "r"(a), "r"(b)); + + return _res; +} + +static inline uint64 +pg_clmulh(uint64 a, uint64 b) +{ + uint64 _res; + + __asm__( + " clmulh %0, %1, %2" +: "=r"(_res) +: "r"(a), "r"(b)); + + return _res; +} + +static inline V128 +pg_clmul128(uint64 a, uint64 b) +{ + V128 result; + + result.lo = pg_clmul(a, b); + result.hi = pg_clmulh(a, b); + return result; +} + +/* + * 128-bit operations + */ +static inline V128 +pg_v128_xor(V128 a, V128 b) +{ + V128 result; + + result.lo = a.lo ^ b.lo; + result.hi = a.hi ^ b.hi; + return result; +} + +static inline V128 +pg_v128_and_mask32(V128 a) +{ + V128 result; + + result.lo = a.lo & UINT64CONST(0x00000000FFFFFFFF); + result.hi = a.hi & UINT64CONST(0x00000000FFFFFFFF); + return result; +} + +static inline V128 +pg_v128_shift_right64(V128 a) +{ + V128 result; + + result.lo = a.hi; + result.hi = 0; + return result; +} + +static inline V128 +pg_v128_shift_right32(V128 a) +{ + V128 result; + + result.lo = (a.lo >> 32) | (a.hi << 32); + result.hi = (a.hi >> 32); + return result; +} + +static inline V128 +pg_v128_load(const unsigned char *p) +{ + V128 result; + + /* + * Load 16 bytes as two 64-bit values. Use direct loads like Abseil + * reference implementation. RISC-V is always little-endian so no byte + * swapping needed. + */ + result.lo = *(const uint64 *) p; + result.hi = *(const uint64 *) (p + 8); + return result; +} + +/* + * CRC-32C (Castagnoli) polynomial folding constants. These are computed + * for the polynomial 0x1EDC6F41 (normal form) or 0x82F63B78 (reflected). + */ +static const uint64 kK5 = UINT64CONST(0x0f20c0dfe); /* Folding constant */ +static const uint64 kK6 = UINT64CONST(0x14cd00bd6); /* Folding constant */ +static const uint64 kK7 = UINT64CONST(0x0dd45aab8); /* 64->32 reduction */ +static const uint64 kP1 = UINT64CONST(0x105ec76f0); /* Barrett reduction */ +static const uint64 kP2 = UINT64CONST(0x0dea713f1); /* Barrett reduction */ + +/* + * Core CRC-32C computation using carry-less multiplication. + * + * Input: CRC in working form (already inverted with ~crc) + * Output: CRC in working form (still inverted) + * + * Precondition: len >= 32 and len % 16 == 0 + */ +static uint32 +pg_crc32c_clmul_core(uint32 crc_inverted, const unsigned char *buf, uint64 len) +{ + V128 x; + + /* Load first 16-byte block and XOR with inverted CRC */ + x = pg_v128_load(buf); + x.lo ^= (uint64) crc_inverted; + buf += 16; + len -= 16; + + /* Fold 16-byte blocks into 128-bit accumulator */ + while (len >= 16) + { + V128 block = pg_v128_load(buf); + V128 lo = pg_clmul128(x.lo, kK5); + V128 hi = pg_clmul128(x.hi, kK6); + + x = pg_v128_xor(pg_v128_xor(lo, hi), block); + buf += 16; + len -= 16; + } + + /* Reduce 128-bit to 64-bit */ + { + V128 tmp = pg_clmul128(kK6, x.lo); + + x = pg_v128_xor(pg_v128_shift_right64(x), tmp); + } + + /* Reduce 64-bit to 32-bit */ + { + V128 tmp = pg_v128_shift_right32(x); + + x = pg_v128_and_mask32(x); + x = pg_clmul128(kK7, x.lo); + x = pg_v128_xor(x, tmp); + } + + /* Barrett reduction to final 32-bit CRC */ + { + V128 tmp = pg_v128_and_mask32(x); + + tmp = pg_clmul128(kP2, tmp.lo); + tmp = pg_v128_and_mask32(tmp); + tmp = pg_clmul128(kP1, tmp.lo); + x = pg_v128_xor(x, tmp); + } + + /* Extract result from second 32-bit lane */ + return (uint32) ((x.lo >> 32) & UINT64CONST(0xFFFFFFFF)); +} + +/* + * Main CRC-32C computation function with RISC-V Zbc acceleration + */ +pg_crc32c +pg_comp_crc32c_riscv_zbc(pg_crc32c crc, const void *data, size_t len) +{ + const unsigned char *p = data; + const size_t kMinLen = 32; + const size_t kChunkLen = 16; + size_t tail; + + /* Use software fallback for small buffers */ + if (len < kMinLen) + return pg_comp_crc32c_sb8(crc, data, len); + + /* + * Process head bytes to align to 16-byte boundary if needed. The hardware + * algorithm requires 16-byte aligned access. + */ + /* Process tail bytes with software (Abseil approach) */ + tail = len % kChunkLen; + if (tail) + { + crc = pg_comp_crc32c_sb8(crc, p, tail); + p += tail; + len -= tail; + } + + /* + * Process remaining bytes (now a multiple of 16) with hardware. The core + * algorithm requires at least 32 bytes. + */ + if (len >= 32) + { + /* + * The Abseil core algorithm expects to receive 0xFFFFFFFF as the + * initial CRC value (corresponding to Abseil's initial value of 0 + * after inversion). PostgreSQL's convention already passes 0xFFFFFFFF + * initially, so pass it directly. The core returns a value that needs + * final XOR with 0xFFFFFFFF (done by the caller). + */ + crc = pg_crc32c_clmul_core(crc, p, len); + } + + return crc; +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 112653c1680..49baf0bad39 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3330,6 +3330,7 @@ VirtualTransactionId VirtualTupleTableSlot VolatileFunctionStatus Vsrt +V128 WAIT_ORDER WALAvailability WALInsertLock -- 2.51.2