From 7eac4aecb0e608180685a7215fda7c06f200c1a1 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Tue, 2 Jul 2024 22:31:42 +1200 Subject: [PATCH v1 2/4] Redefine port/atomics.h on top of . * Redirect types and functions to standard C/C++ * Use instead for C++11 extensions (as formalized by C++23) * Add pg_atomic_uint8, pg_atomic_uint16 and pg_atomic(T) * Provide generic pg_atomic_read(), pg_atomic_write() * Assume that most basic operations will be called by standardized generics in future code, eg atomic_fetch_add() * Forward all the traditional _32, _64 function names for existing code * Reimplement pg_atomic_flag with pg_atomic(uint8) to allow relaxed load (impossible with standard atomic_flag, and pg_atomic(bool) doesn't seem to be portable enough) * Let the compiler/runtime emulate 64 bit atomics with locks for armv7 * The per-compiler headers now just define pg_compiler_barrier_impl() * The per-arch headers now optionally define pg_{memory,read,write}_barrier_impl() but only if they think they can do something better than compilers (to investigate...) * Drop the secondary copy of the ancient spinlock ops from atomics * Drop arch-{arm,ppc}.h, fallback.h and generic.h WIP! Need to research generated code on all architectures and see if anything important is lost. Need to think about whether alignment assertions are worth saving. Need to test on Visual Studio 2022 with /experimental:c11atomics (?). --- src/include/port/atomics.h | 621 ++++++------------------ src/include/port/atomics/arch-arm.h | 32 -- src/include/port/atomics/arch-ppc.h | 256 ---------- src/include/port/atomics/arch-x86.h | 205 -------- src/include/port/atomics/fallback.h | 42 -- src/include/port/atomics/generic-gcc.h | 307 +----------- src/include/port/atomics/generic-msvc.h | 98 +--- src/include/port/atomics/generic.h | 427 ---------------- src/tools/pgindent/typedefs.list | 2 + 9 files changed, 166 insertions(+), 1824 deletions(-) delete mode 100644 src/include/port/atomics/arch-arm.h delete mode 100644 src/include/port/atomics/arch-ppc.h delete mode 100644 src/include/port/atomics/fallback.h delete mode 100644 src/include/port/atomics/generic.h diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index 96f1858da97..61a6b4322fd 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -3,24 +3,33 @@ * atomics.h * Atomic operations. * - * Hardware and compiler dependent functions for manipulating memory - * atomically and dealing with cache coherency. Used to implement locking - * facilities and lockless algorithms/data structures. + * These interfaces are for manipulating memory atomically and dealing with + * cache coherency. They can be used to implement locking facilities and + * lockless algorithms/data structures. * - * To bring up postgres on a platform/compiler at the very least - * implementations for the following operations should be provided: - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() - * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. + * This is mostly a renaming of , with some differences: * - * There exist generic, hardware independent, implementations for several - * compilers which might be sufficient, although possibly not optimal, for a - * new platform. If no such generic implementation is available spinlocks will - * be used to implement the 64-bit parts of the API. + * * read/write barriers (weaker than acquire/release) + * * they affect even non-atomic access without dependency on atomic access + * * pg_atomic_flag semantics don't allow mapping to atomic_flag * - * Implement _u64 atomics if and only if your platform can use them - * efficiently (and obviously correctly). + * PostgreSQL atomic type width assumptions: + * + * 8, 16: Required to be lock-free (even though implemented with wider + * instructions by the compiler on some RISC-V systems). + * + * 32: Required to be lock-free. No known modern system lacks them. + * + * 64: These can reasonably be expected to be lock-free and fast on + * all modern-ish systems. For one known low-end system they are + * emulated by the compiler/runtime with locks (armv7). + * + * In all cases values must be well aligned or undefined behavior results. + * + * To bring up postgres on a compiler, provide: + * * pg_compiler_barrier() + * Optionally also: + * * pg_memory_barrier(), pg_write_barrier(), pg_read_barrier() * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -38,48 +47,59 @@ #ifndef ATOMICS_H #define ATOMICS_H -#ifdef FRONTEND -#error "atomics.h may not be included from frontend code" -#endif - #define INSIDE_ATOMICS_H -#include +#if defined(__cplusplus) && __cplusplus < 202302L +/* C++11 couldn't handle . Use approach from C++23 33.5.12. */ +extern "C++" +{ +#include +#define pg_atomic(T) std::atomic + using std::memory_order_relaxed; + using std::memory_order_acquire; + using std::memory_order_release; + using std::memory_order_acq_rel; + using std::memory_order_seq_cst; +} +#else +#include +#define pg_atomic(T) _Atomic(T) +#endif + +/* Assumptions about lock-free access. */ +StaticAssertDecl(ATOMIC_CHAR_LOCK_FREE >= 2, "need lock-free 8-bit atomics"); +StaticAssertDecl(ATOMIC_SHORT_LOCK_FREE >= 2, "need lock-free 16-bit atomics"); +StaticAssertDecl(ATOMIC_INT_LOCK_FREE >= 2, "need lock-free 32-bit atomics"); + +/* Can't use standard atomic_flag due to extended semantics, see below. */ +typedef pg_atomic(uint8) +pg_atomic_flag; + +/* Common supported atomic integer types. */ +typedef pg_atomic(uint8) +pg_atomic_uint8; +typedef pg_atomic(uint16) +pg_atomic_uint16; +typedef pg_atomic(uint32) +pg_atomic_uint32; +typedef pg_atomic(uint64) +pg_atomic_uint64; /* * First a set of architecture specific files is included. * - * These files can provide the full set of atomics or can do pretty much - * nothing if all the compilers commonly used on these platforms provide - * usable generics. - * - * Don't add an inline assembly of the actual atomic operations if all the - * common implementations of your platform provide intrinsics. Intrinsics are - * much easier to understand and potentially support more architectures. - * * It will often make sense to define memory barrier semantics here, since * e.g. generic compiler intrinsics for x86 memory barriers can't know that * postgres doesn't need x86 read/write barriers do anything more than a * compiler barrier. - * */ -#if defined(__arm__) || defined(__arm) || defined(__aarch64__) -#include "port/atomics/arch-arm.h" -#elif defined(__i386__) || defined(__i386) || defined(__x86_64__) +#if defined(__i386__) || defined(__i386) || defined(__x86_64__) #include "port/atomics/arch-x86.h" -#elif defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) -#include "port/atomics/arch-ppc.h" #endif /* * Compiler specific, but architecture independent implementations. - * - * Provide architecture independent implementations of the atomic - * facilities. At the very least compiler barriers should be provided, but a - * full implementation of - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * using compiler intrinsics are a good idea. + * This defines compiler barriers. */ /* * gcc or compatible, including clang and icc. @@ -93,29 +113,9 @@ #endif /* Fail if we couldn't find implementations of required facilities. */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#error "could not find an implementation of pg_atomic_uint32" -#endif #if !defined(pg_compiler_barrier_impl) #error "could not find an implementation of pg_compiler_barrier" #endif -#if !defined(pg_memory_barrier_impl) -#error "could not find an implementation of pg_memory_barrier_impl" -#endif - - -/* - * Provide a spinlock-based implementation of the 64 bit variants, if - * necessary. - */ -#include "port/atomics/fallback.h" - -/* - * Provide additional operations using supported infrastructure. These are - * expected to be efficient if the underlying atomic operations are efficient. - */ -#include "port/atomics/generic.h" - /* * pg_compiler_barrier - prevent the compiler from moving code across @@ -125,6 +125,10 @@ * reorder loads or stores to main memory around the barrier. However, the * CPU may still reorder loads or stores at runtime, if the architecture's * memory model permits this. + * + * Unlike standard atomic_signal_fence(), pg_compiler_barrier() prevents + * reordering of non-atomic, non-volatile accesses, so a compiler-specific + * definition is required. */ #define pg_compiler_barrier() pg_compiler_barrier_impl() @@ -137,8 +141,20 @@ * loads and stores are totally ordered (which is not the case on most * architectures) this requires issuing some sort of memory fencing * instruction. + * + * Unlike standard atomic_thread_fence(), pg_{read,write}_barrier() affects + * non-atomic, non-volatile accesses. */ -#define pg_memory_barrier() pg_memory_barrier_impl() +static inline void +pg_memory_barrier(void) +{ +#ifdef pg_memory_barrier_impl + pg_memory_barrier_impl(); +#else + pg_compiler_barrier(); + atomic_thread_fence(memory_order_seq_cst); +#endif +} /* * pg_(read|write)_barrier - prevent the CPU from reordering memory access @@ -150,427 +166,116 @@ * are thus weaker than a full memory barrier, but stronger than a compiler * barrier. In practice, on machines with strong memory ordering, read and * write barriers may require nothing more than a compiler barrier. - */ -#define pg_read_barrier() pg_read_barrier_impl() -#define pg_write_barrier() pg_write_barrier_impl() - -/* - * Spinloop delay - Allow CPU to relax in busy loops - */ -#define pg_spin_delay() pg_spin_delay_impl() - -/* - * pg_atomic_init_flag - initialize atomic flag. - * - * No barrier semantics. - */ -static inline void -pg_atomic_init_flag(volatile pg_atomic_flag *ptr) -{ - pg_atomic_init_flag_impl(ptr); -} - -/* - * pg_atomic_test_set_flag - TAS() - * - * Returns true if the flag has successfully been set, false otherwise. - * - * Acquire (including read barrier) semantics. - */ -static inline bool -pg_atomic_test_set_flag(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_test_set_flag_impl(ptr); -} - -/* - * pg_atomic_unlocked_test_flag - Check if the lock is free - * - * Returns true if the flag currently is not set, false otherwise. - * - * No barrier semantics. - */ -static inline bool -pg_atomic_unlocked_test_flag(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_unlocked_test_flag_impl(ptr); -} - -/* - * pg_atomic_clear_flag - release lock set by TAS() * - * Release (including write barrier) semantics. + * Unlike standard atomic_thread_fence(), pg_{read,write}_barrier() affects + * non-atomic, non-volatile accesses. The default implementation uses + * acquire/release fences, but these are strictly stronger than read/write. */ -static inline void -pg_atomic_clear_flag(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} - -/* - * pg_atomic_init_u32 - initialize atomic variable - * - * Has to be done before any concurrent usage.. - * - * No barrier semantics. - */ static inline void -pg_atomic_init_u32(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - AssertPointerAlignment(ptr, 4); - - pg_atomic_init_u32_impl(ptr, val); -} - -/* - * pg_atomic_read_u32 - unlocked read from atomic variable. - * - * The read is guaranteed to return a value as it has been written by this or - * another process at some point in the past. There's however no cache - * coherency interaction guaranteeing the value hasn't since been written to - * again. - * - * No barrier semantics. - */ -static inline uint32 -pg_atomic_read_u32(volatile pg_atomic_uint32 *ptr) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_read_u32_impl(ptr); -} - -/* - * pg_atomic_read_membarrier_u32 - read with barrier semantics. - * - * This read is guaranteed to return the current value, provided that the value - * is only ever updated via operations with barrier semantics, such as - * pg_atomic_compare_exchange_u32() and pg_atomic_write_membarrier_u32(). - * While this may be less performant than pg_atomic_read_u32(), it may be - * easier to reason about correctness with this function in less performance- - * sensitive code. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_read_membarrier_u32(volatile pg_atomic_uint32 *ptr) +pg_read_barrier(void) { - AssertPointerAlignment(ptr, 4); - - return pg_atomic_read_membarrier_u32_impl(ptr); -} - -/* - * pg_atomic_write_u32 - write to atomic variable. - * - * The write is guaranteed to succeed as a whole, i.e. it's not possible to - * observe a partial write for any reader. Note that this correctly interacts - * with pg_atomic_compare_exchange_u32, in contrast to - * pg_atomic_unlocked_write_u32(). - * - * No barrier semantics. - */ -static inline void -pg_atomic_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - AssertPointerAlignment(ptr, 4); - - pg_atomic_write_u32_impl(ptr, val); -} - -/* - * pg_atomic_unlocked_write_u32 - unlocked write to atomic variable. - * - * The write is guaranteed to succeed as a whole, i.e. it's not possible to - * observe a partial write for any reader. But note that writing this way is - * not guaranteed to correctly interact with read-modify-write operations like - * pg_atomic_compare_exchange_u32. This should only be used in cases where - * minor performance regressions due to atomics emulation are unacceptable. - * - * No barrier semantics. - */ -static inline void -pg_atomic_unlocked_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - AssertPointerAlignment(ptr, 4); - - pg_atomic_unlocked_write_u32_impl(ptr, val); +#ifdef pg_read_barrier_impl + pg_read_barrier_impl(); +#else + pg_compiler_barrier(); + atomic_thread_fence(memory_order_acquire); +#endif } -/* - * pg_atomic_write_membarrier_u32 - write with barrier semantics. - * - * The write is guaranteed to succeed as a whole, i.e., it's not possible to - * observe a partial write for any reader. Note that this correctly interacts - * with both pg_atomic_compare_exchange_u32() and - * pg_atomic_read_membarrier_u32(). While this may be less performant than - * pg_atomic_write_u32(), it may be easier to reason about correctness with - * this function in less performance-sensitive code. - * - * Full barrier semantics. - */ static inline void -pg_atomic_write_membarrier_u32(volatile pg_atomic_uint32 *ptr, uint32 val) +pg_write_barrier(void) { - AssertPointerAlignment(ptr, 4); - - pg_atomic_write_membarrier_u32_impl(ptr, val); -} - -/* - * pg_atomic_exchange_u32 - exchange newval with current value - * - * Returns the old value of 'ptr' before the swap. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_exchange_u32(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - AssertPointerAlignment(ptr, 4); - - return pg_atomic_exchange_u32_impl(ptr, newval); +#ifdef pg_write_barrier_impl + pg_write_barrier_impl(); +#else + pg_compiler_barrier(); + atomic_thread_fence(memory_order_release); +#endif } /* - * pg_atomic_compare_exchange_u32 - CAS operation - * - * Atomically compare the current value of ptr with *expected and store newval - * iff ptr and *expected have the same value. The current value of *ptr will - * always be stored in *expected. - * - * Return true if values have been exchanged, false otherwise. + * Operations corresponding to standard atomic_flag. We have to use an integer + * instead of mapping directly to the standard names, to support our relaxed + * check function. * - * Full barrier semantics. + * Acquire and release fences, which are respectively stronger than read and + * write barriers. */ -static inline bool -pg_atomic_compare_exchange_u32(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - AssertPointerAlignment(ptr, 4); - AssertPointerAlignment(expected, 4); - - return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval); -} +#define pg_atomic_init_flag(p) atomic_init((p), 1) +#define pg_atomic_test_set_flag(p) \ + atomic_fetch_and_explicit((p), 0, memory_order_acquire) +#define pg_atomic_clear_flag(p) \ + atomic_store_explicit((p), 1, memory_order_release) +#define pg_atomic_unlocked_test_flag(p) \ + atomic_load_explicit((p), memory_order_relaxed) /* - * pg_atomic_fetch_add_u32 - atomically add to variable - * - * Returns the value of ptr before the arithmetic operation. - * - * Full barrier semantics. + * Local convention for load/store, relaxed AKA no barrier. These are the only + * "generic" functions provided with pg_ prefixes. It seems pointless to + * rename everything in , but these two are PostgreSQL's + * established way of representing relaxed access, and a lot shorter. */ -static inline uint32 -pg_atomic_fetch_add_u32(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_fetch_add_u32_impl(ptr, add_); -} +#define pg_atomic_read(p) \ + atomic_load_explicit((p), memory_order_relaxed) +#define pg_atomic_write(p, v) \ + atomic_store_explicit((p), (v), memory_order_relaxed) /* - * pg_atomic_fetch_sub_u32 - atomically subtract from variable + * Backward-compatible type-specific function names. Only the historical _u32 + * and _64 names are provided. New code and code using other atomic types + * should use the generic functions from the standard directly, and + * pg_atomic_{read,write}() for relaxed access. * - * Returns the value of ptr before the arithmetic operation. Note that sub_ - * may not be INT_MIN due to platform limitations. - * - * Full barrier semantics. + * All of these except pg_atomic_(unlocked_){read,write}_{32,64} have seq cst + * AKA full barrier semantics. */ -static inline uint32 -pg_atomic_fetch_sub_u32(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - AssertPointerAlignment(ptr, 4); - Assert(sub_ != INT_MIN); - return pg_atomic_fetch_sub_u32_impl(ptr, sub_); -} +#define pg_atomic_init_u32 atomic_init +#define pg_atomic_init_u64 atomic_init +#define pg_atomic_read_u32 pg_atomic_read +#define pg_atomic_read_u64 pg_atomic_read +#define pg_atomic_write_u32 pg_atomic_write +#define pg_atomic_write_u64 pg_atomic_write +#define pg_atomic_unlocked_read_u32 pg_atomic_read +#define pg_atomic_unlocked_read_u64 pg_atomic_read +#define pg_atomic_unlocked_write_u32 pg_atomic_write +#define pg_atomic_unlocked_write_u64 pg_atomic_write +#define pg_atomic_read_membarrier_u32 atomic_load +#define pg_atomic_read_membarrier_u64 atomic_load +#define pg_atomic_write_membarrier_u32 atomic_store +#define pg_atomic_write_membarrier_u64 atomic_store +#define pg_atomic_exchange_u32 atomic_exchange +#define pg_atomic_exchange_u64 atomic_exchange +#define pg_atomic_compare_exchange_u32 atomic_compare_exchange_strong +#define pg_atomic_compare_exchange_u64 atomic_compare_exchange_strong +#define pg_atomic_fetch_add_u32 atomic_fetch_add +#define pg_atomic_fetch_add_u64 atomic_fetch_add +#define pg_atomic_fetch_sub_u32 atomic_fetch_sub +#define pg_atomic_fetch_sub_u64 atomic_fetch_sub +#define pg_atomic_fetch_and_u32 atomic_fetch_and +#define pg_atomic_fetch_and_u64 atomic_fetch_and +#define pg_atomic_fetch_or_u32 atomic_fetch_or +#define pg_atomic_fetch_or_u64 atomic_fetch_or /* - * pg_atomic_fetch_and_u32 - atomically bit-and and_ with variable - * - * Returns the value of ptr before the arithmetic operation. - * - * Full barrier semantics. + * The rest of this file defines non-fundamental convenience functions with no + * counterpart in . */ -static inline uint32 -pg_atomic_fetch_and_u32(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_fetch_and_u32_impl(ptr, and_); -} /* - * pg_atomic_fetch_or_u32 - atomically bit-or or_ with variable - * - * Returns the value of ptr before the arithmetic operation. - * - * Full barrier semantics. + * Fetch and add/subtract wrappers that return the new value instead of the old + * value. We need functions to avoid double evaluation of v. */ -static inline uint32 -pg_atomic_fetch_or_u32(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_fetch_or_u32_impl(ptr, or_); -} - -/* - * pg_atomic_add_fetch_u32 - atomically add to variable - * - * Returns the value of ptr after the arithmetic operation. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_add_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_add_fetch_u32_impl(ptr, add_); -} - -/* - * pg_atomic_sub_fetch_u32 - atomically subtract from variable - * - * Returns the value of ptr after the arithmetic operation. Note that sub_ may - * not be INT_MIN due to platform limitations. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_sub_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - AssertPointerAlignment(ptr, 4); - Assert(sub_ != INT_MIN); - return pg_atomic_sub_fetch_u32_impl(ptr, sub_); -} - -/* ---- - * The 64 bit operations have the same semantics as their 32bit counterparts - * if they are available. Check the corresponding 32bit function for - * documentation. - * ---- - */ -static inline void -pg_atomic_init_u64(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * Can't necessarily enforce alignment - and don't need it - when using - * the spinlock based fallback implementation. Therefore only assert when - * not using it. - */ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - pg_atomic_init_u64_impl(ptr, val); -} - -static inline uint64 -pg_atomic_read_u64(volatile pg_atomic_uint64 *ptr) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_read_u64_impl(ptr); -} - -static inline uint64 -pg_atomic_read_membarrier_u64(volatile pg_atomic_uint64 *ptr) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_read_membarrier_u64_impl(ptr); -} - -static inline void -pg_atomic_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - pg_atomic_write_u64_impl(ptr, val); -} - -static inline void -pg_atomic_write_membarrier_u64(volatile pg_atomic_uint64 *ptr, uint64 val) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - pg_atomic_write_membarrier_u64_impl(ptr, val); -} - -static inline uint64 -pg_atomic_exchange_u64(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_exchange_u64_impl(ptr, newval); -} - -static inline bool -pg_atomic_compare_exchange_u64(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval); -} - -static inline uint64 -pg_atomic_fetch_add_u64(volatile pg_atomic_uint64 *ptr, int64 add_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_fetch_add_u64_impl(ptr, add_); -} - -static inline uint64 -pg_atomic_fetch_sub_u64(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - Assert(sub_ != PG_INT64_MIN); - return pg_atomic_fetch_sub_u64_impl(ptr, sub_); -} - -static inline uint64 -pg_atomic_fetch_and_u64(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_fetch_and_u64_impl(ptr, and_); -} - -static inline uint64 -pg_atomic_fetch_or_u64(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_fetch_or_u64_impl(ptr, or_); -} - -static inline uint64 -pg_atomic_add_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 add_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_add_fetch_u64_impl(ptr, add_); -} - -static inline uint64 -pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - Assert(sub_ != PG_INT64_MIN); - return pg_atomic_sub_fetch_u64_impl(ptr, sub_); +#define PG_ATOMIC_GEN_REVERSE_FETCH(size, name, op) \ +static inline uint##size \ +pg_atomic_##name##_fetch_u##size(volatile pg_atomic_uint##size *p, uint##size v) \ +{ \ + return atomic_fetch_##name(p, v) op v; \ } +PG_ATOMIC_GEN_REVERSE_FETCH(32, add, +); +PG_ATOMIC_GEN_REVERSE_FETCH(64, add, +); +PG_ATOMIC_GEN_REVERSE_FETCH(32, sub, -); +PG_ATOMIC_GEN_REVERSE_FETCH(64, sub, -); /* * Monotonically advance the given variable using only atomic operations until @@ -584,11 +289,7 @@ pg_atomic_monotonic_advance_u64(volatile pg_atomic_uint64 *ptr, uint64 target) { uint64 currval; -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - - currval = pg_atomic_read_u64_impl(ptr); + currval = pg_atomic_read_u64(ptr); if (currval >= target) { pg_memory_barrier(); diff --git a/src/include/port/atomics/arch-arm.h b/src/include/port/atomics/arch-arm.h deleted file mode 100644 index eb7e0ac93ba..00000000000 --- a/src/include/port/atomics/arch-arm.h +++ /dev/null @@ -1,32 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-arm.h - * Atomic operations considerations specific to ARM - * - * Portions Copyright (c) 2013-2025, PostgreSQL Global Development Group - * - * NOTES: - * - * src/include/port/atomics/arch-arm.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * 64 bit atomics on ARM32 are implemented using kernel fallbacks and thus - * might be slow, so disable entirely. On ARM64 that problem doesn't exist. - */ -#if !defined(__aarch64__) -#define PG_DISABLE_64_BIT_ATOMICS -#else -/* - * Architecture Reference Manual for ARMv8 states aligned read/write to/from - * general purpose register is atomic. - */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* __aarch64__ */ diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h deleted file mode 100644 index b93f6766d29..00000000000 --- a/src/include/port/atomics/arch-ppc.h +++ /dev/null @@ -1,256 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-ppc.h - * Atomic operations considerations specific to PowerPC - * - * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-ppc.h - * - *------------------------------------------------------------------------- - */ - -#if defined(__GNUC__) - -/* - * lwsync orders loads with respect to each other, and similarly with stores. - * But a load can be performed before a subsequent store, so sync must be used - * for a full memory barrier. - */ -#define pg_memory_barrier_impl() __asm__ __volatile__ ("sync" : : : "memory") -#define pg_read_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#define pg_write_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* 64bit atomics are only supported in 64bit mode */ -#if SIZEOF_VOID_P >= 8 -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - volatile uint64 value pg_attribute_aligned(8); -} pg_atomic_uint64; - -#endif - -/* - * This mimics gcc __atomic_compare_exchange_n(..., __ATOMIC_SEQ_CST), but - * code generation differs at the end. __atomic_compare_exchange_n(): - * 100: isync - * 104: mfcr r3 - * 108: rlwinm r3,r3,3,31,31 - * 10c: bne 120 <.eb+0x10> - * 110: clrldi r3,r3,63 - * 114: addi r1,r1,112 - * 118: blr - * 11c: nop - * 120: clrldi r3,r3,63 - * 124: stw r9,0(r4) - * 128: addi r1,r1,112 - * 12c: blr - * - * This: - * f0: isync - * f4: mfcr r9 - * f8: rldicl. r3,r9,35,63 - * fc: bne 104 <.eb> - * 100: stw r10,0(r4) - * 104: addi r1,r1,112 - * 108: blr - * - * This implementation may or may not have materially different performance. - * It's not exploiting the fact that cr0 still holds the relevant comparison - * bits, set during the __asm__. One could fix that by moving more code into - * the __asm__. (That would remove the freedom to eliminate dead stores when - * the caller ignores "expected", but few callers do.) - * - * Recognizing constant "newval" would be superfluous, because there's no - * immediate-operand version of stwcx. - */ -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - uint32 found; - uint32 condition_register; - bool ret; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int32) *expected <= PG_INT16_MAX && - (int32) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpwi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpw %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -/* - * This mirrors gcc __sync_fetch_and_add(). - * - * Like tas(), use constraint "=&b" to avoid allocating r0. - */ -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 _t; - uint32 res; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#ifdef PG_HAVE_ATOMIC_U64_SUPPORT - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - uint64 found; - uint32 condition_register; - bool ret; - - AssertPointerAlignment(expected, 8); - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/; s/cmpw/cmpd/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int64) *expected <= PG_INT16_MAX && - (int64) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpdi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpd %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 _t; - uint64 res; - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ - -/* per architecture manual doubleword accesses have single copy atomicity */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY diff --git a/src/include/port/atomics/arch-x86.h b/src/include/port/atomics/arch-x86.h index 4ba2ccc0591..acd612099fa 100644 --- a/src/include/port/atomics/arch-x86.h +++ b/src/include/port/atomics/arch-x86.h @@ -3,10 +3,6 @@ * arch-x86.h * Atomic operations considerations specific to intel x86 * - * Note that we actually require a 486 upwards because the 386 doesn't have - * support for xadd and cmpxchg. Given that the 386 isn't supported anywhere - * anymore that's not much of a restriction luckily. - * * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * @@ -43,204 +39,3 @@ #define pg_read_barrier_impl() pg_compiler_barrier_impl() #define pg_write_barrier_impl() pg_compiler_barrier_impl() - -/* - * Provide implementation for atomics using inline assembly on x86 gcc. It's - * nice to support older gcc's and the compare/exchange implementation here is - * actually more efficient than the * __sync variant. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - volatile char value; -} pg_atomic_flag; - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* - * It's too complicated to write inline asm for 64bit types on 32bit and the - * 486 can't do it anyway. - */ -#ifdef __x86_64__ -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - /* alignment guaranteed due to being on a 64bit platform */ - volatile uint64 value; -} pg_atomic_uint64; -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -#if !defined(PG_HAVE_SPIN_DELAY) -/* - * This sequence is equivalent to the PAUSE instruction ("rep" is - * ignored by old IA32 processors if the following instruction is - * not a string operation); the IA-32 Architecture Software - * Developer's Manual, Vol. 3, Section 7.7.2 describes why using - * PAUSE in the inner loop of a spin lock is necessary for good - * performance: - * - * The PAUSE instruction improves the performance of IA-32 - * processors supporting Hyper-Threading Technology when - * executing spin-wait loops and other routines where one - * thread is accessing a shared lock or semaphore in a tight - * polling loop. When executing a spin-wait loop, the - * processor can suffer a severe performance penalty when - * exiting the loop because it detects a possible memory order - * violation and flushes the core processor's pipeline. The - * PAUSE instruction provides a hint to the processor that the - * code sequence is a spin-wait loop. The processor uses this - * hint to avoid the memory order violation and prevent the - * pipeline flush. In addition, the PAUSE instruction - * de-pipelines the spin-wait loop to prevent it from - * consuming execution resources excessively. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#define PG_HAVE_SPIN_DELAY -static __inline__ void -pg_spin_delay_impl(void) -{ - __asm__ __volatile__(" rep; nop \n"); -} -#elif defined(_MSC_VER) && defined(__x86_64__) -#define PG_HAVE_SPIN_DELAY -static __forceinline void -pg_spin_delay_impl(void) -{ - _mm_pause(); -} -#elif defined(_MSC_VER) -#define PG_HAVE_SPIN_DELAY -static __forceinline void -pg_spin_delay_impl(void) -{ - /* See comment for gcc code. Same code, MASM syntax */ - __asm rep nop; -} -#endif -#endif /* !defined(PG_HAVE_SPIN_DELAY) */ - - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - char _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(ptr->value) -: -: "memory"); - return _res == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* - * On a TSO architecture like x86 it's sufficient to use a compiler - * barrier to achieve release semantics. - */ - __asm__ __volatile__("" ::: "memory"); - ptr->value = 0; -} - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - char ret; - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgl %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 res; - __asm__ __volatile__( - " lock \n" - " xaddl %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#ifdef __x86_64__ - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - char ret; - - AssertPointerAlignment(expected, 8); - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgq %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 res; - __asm__ __volatile__( - " lock \n" - " xaddq %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -/* - * 8 byte reads / writes have single-copy atomicity on 32 bit x86 platforms - * since at least the 586. As well as on all x86-64 cpus. - */ -#if defined(__i568__) || defined(__i668__) || /* gcc i586+ */ \ - (defined(_M_IX86) && _M_IX86 >= 500) || /* msvc i586+ */ \ - defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) /* gcc, msvc */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* 8 byte single-copy atomicity */ diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h deleted file mode 100644 index d2f1b851668..00000000000 --- a/src/include/port/atomics/fallback.h +++ /dev/null @@ -1,42 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fallback.h - * Fallback for platforms without 64 bit atomics support. Slower - * than native atomics support, but not unusably slow. - * - * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/fallback.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - - -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) - -#define PG_HAVE_ATOMIC_U64_SIMULATION - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - int sema; - volatile uint64 value; -} pg_atomic_uint64; - -#define PG_HAVE_ATOMIC_INIT_U64 -extern void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_); - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval); - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_); - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h index a0751f2286a..78555a27861 100644 --- a/src/include/port/atomics/generic-gcc.h +++ b/src/include/port/atomics/generic-gcc.h @@ -1,19 +1,11 @@ /*------------------------------------------------------------------------- * * generic-gcc.h - * Atomic operations, implemented using gcc (or compatible) intrinsics. + * Compiler barriers for GCC (or compatible) * * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * NOTES: - * - * Documentation: - * * Legacy __sync Built-in Functions for Atomic Memory Access - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html - * * Built-in functions for memory model aware atomic operations - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html - * * src/include/port/atomics/generic-gcc.h * *------------------------------------------------------------------------- @@ -28,300 +20,3 @@ * An empty asm block should be a sufficient compiler barrier. */ #define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory") - -/* - * If we're on GCC, we should be able to get a memory barrier - * out of this compiler built-in. But we prefer to rely on platform specific - * definitions where possible, and use this only as a fallback. - */ -#if !defined(pg_memory_barrier_impl) -# if defined(HAVE_GCC__ATOMIC_INT32_CAS) -# define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST) -# elif defined(__GNUC__) -# define pg_memory_barrier_impl() __sync_synchronize() -# endif -#endif /* !defined(pg_memory_barrier_impl) */ - -#if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* acquire semantics include read barrier semantics */ -# define pg_read_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_ACQUIRE); \ -} while (0) -#endif - -#if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* release semantics include write barrier semantics */ -# define pg_write_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_RELEASE); \ -} while (0) -#endif - - -/* generic gcc based atomic flag implementation */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \ - && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS)) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - /* - * If we have a choice, use int-width TAS, because that is more efficient - * and/or more reliably implemented on most non-Intel platforms. (Note - * that this code isn't used on x86[_64]; see arch-x86.h for that.) - */ -#ifdef HAVE_GCC__SYNC_INT32_TAS - volatile int value; -#else - volatile char value; -#endif -} pg_atomic_flag; - -#endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */ - -/* generic gcc based atomic uint32 implementation */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \ - && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS)) - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */ - -/* generic gcc based atomic uint64 implementation */ -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \ - && !defined(PG_DISABLE_64_BIT_ATOMICS) \ - && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS)) - -#define PG_HAVE_ATOMIC_U64_SUPPORT - -typedef struct pg_atomic_uint64 -{ - volatile uint64 value pg_attribute_aligned(8); -} pg_atomic_uint64; - -#endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ - -#ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT - -#if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) - -#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* NB: only an acquire barrier, not a full one */ - /* some platform only support a 1 here */ - return __sync_lock_test_and_set(&ptr->value, 1) == 0; -} -#endif - -#endif /* defined(HAVE_GCC__SYNC_*_TAS) */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return ptr->value == 0; -} -#endif - -#ifndef PG_HAVE_ATOMIC_CLEAR_FLAG -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - __sync_lock_release(&ptr->value); -} -#endif - -#ifndef PG_HAVE_ATOMIC_INIT_FLAG -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} -#endif - -#endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */ - -/* prefer __atomic, it has a better API */ -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - /* FIXME: we can probably use a lower consistency model */ - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 32-bit __atomic_compare_exchange_n() implies - * the availability of 32-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - - -#if !defined(PG_DISABLE_64_BIT_ATOMICS) - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - AssertPointerAlignment(expected, 8); - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - - AssertPointerAlignment(expected, 8); - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 64-bit __atomic_compare_exchange_n() implies - * the availability of 64-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - -#endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */ diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h index a6ea5f1c2e7..4cb5eab5be6 100644 --- a/src/include/port/atomics/generic-msvc.h +++ b/src/include/port/atomics/generic-msvc.h @@ -1,17 +1,11 @@ /*------------------------------------------------------------------------- * * generic-msvc.h - * Atomic operations support when using MSVC + * Compiler barriers when using MSVC * * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * NOTES: - * - * Documentation: - * * Interlocked Variable Access - * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx - * * src/include/port/atomics/generic-msvc.h * *------------------------------------------------------------------------- @@ -24,92 +18,4 @@ #endif #pragma intrinsic(_ReadWriteBarrier) -#define pg_compiler_barrier_impl() _ReadWriteBarrier() - -#ifndef pg_memory_barrier_impl -#define pg_memory_barrier_impl() MemoryBarrier() -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_attribute_aligned(8) pg_atomic_uint64 -{ - volatile uint64 value; -} pg_atomic_uint64; - - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = InterlockedCompareExchange(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return InterlockedExchange(&ptr->value, newval); -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return InterlockedExchangeAdd(&ptr->value, add_); -} - -/* - * The non-intrinsics versions are only available in vista upwards, so use the - * intrinsic version. Only supported on >486, but we require XP as a minimum - * baseline, which doesn't support the 486, so we don't need to add checks for - * that case. - */ -#pragma intrinsic(_InterlockedCompareExchange64) - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - current = _InterlockedCompareExchange64(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -/* Only implemented on 64bit builds */ -#ifdef _WIN64 - -#pragma intrinsic(_InterlockedExchange64) - -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return _InterlockedExchange64(&ptr->value, newval); -} - -#pragma intrinsic(_InterlockedExchangeAdd64) - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return _InterlockedExchangeAdd64(&ptr->value, add_); -} - -#endif /* _WIN64 */ +#define pg_compiler_barrier_impl() _ReadWriteBarrier() diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h deleted file mode 100644 index 6b61a7b5416..00000000000 --- a/src/include/port/atomics/generic.h +++ /dev/null @@ -1,427 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic.h - * Implement higher level operations based on some lower level atomic - * operations. - * - * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/generic.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - -/* - * If read or write barriers are undefined, we upgrade them to full memory - * barriers. - */ -#if !defined(pg_read_barrier_impl) -# define pg_read_barrier_impl pg_memory_barrier_impl -#endif -#if !defined(pg_write_barrier_impl) -# define pg_write_barrier_impl pg_memory_barrier_impl -#endif - -#ifndef PG_HAVE_SPIN_DELAY -#define PG_HAVE_SPIN_DELAY -#define pg_spin_delay_impl() ((void)0) -#endif - - -/* provide fallback */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef pg_atomic_uint32 pg_atomic_flag; -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U32 -#define PG_HAVE_ATOMIC_READ_U32 -static inline uint32 -pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return ptr->value; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U32 -#define PG_HAVE_ATOMIC_WRITE_U32 -static inline void -pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -static inline void -pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -/* - * provide fallback for test_and_set using atomic_exchange if available - */ -#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_exchange_u32_impl(ptr, 1) == 0; -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -/* - * provide fallback for test_and_set using atomic_compare_exchange if - * available. - */ -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - uint32 value = 0; - return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1); -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) -# error "No pg_atomic_test_and_set provided" -#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */ - - -#ifndef PG_HAVE_ATOMIC_INIT_U32 -#define PG_HAVE_ATOMIC_INIT_U32 -static inline void -pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_ADD_FETCH_U32 -static inline uint32 -pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) -#define PG_HAVE_ATOMIC_SUB_FETCH_U32 -static inline uint32 -pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U32 -static inline uint32 -pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return pg_atomic_fetch_add_u32_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32 -static inline void -pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - (void) pg_atomic_exchange_u32_impl(ptr, val); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U64 -#define PG_HAVE_ATOMIC_WRITE_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * On this platform aligned 64bit writes are guaranteed to be atomic, - * except if using the fallback implementation, where can't guarantee the - * required alignment. - */ - AssertPointerAlignment(ptr, 8); - ptr->value = val; -} - -#else - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * 64 bit writes aren't safe on all platforms. In the generic - * implementation implement them as an atomic exchange. - */ - pg_atomic_exchange_u64_impl(ptr, val); -} - -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_WRITE_U64 */ - -#ifndef PG_HAVE_ATOMIC_READ_U64 -#define PG_HAVE_ATOMIC_READ_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - /* - * On this platform aligned 64-bit reads are guaranteed to be atomic. - */ - AssertPointerAlignment(ptr, 8); - return ptr->value; -} - -#else - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - uint64 old = 0; - - /* - * 64-bit reads aren't atomic on all platforms. In the generic - * implementation implement them as a compare/exchange with 0. That'll - * fail or succeed, but always return the old value. Possibly might store - * a 0, but only if the previous value also was a 0 - i.e. harmless. - */ - pg_atomic_compare_exchange_u64_impl(ptr, &old, 0); - - return old; -} -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_READ_U64 */ - -#ifndef PG_HAVE_ATOMIC_INIT_U64 -#define PG_HAVE_ATOMIC_INIT_U64 -static inline void -pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_ADD_FETCH_U64 -static inline uint64 -pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) -#define PG_HAVE_ATOMIC_SUB_FETCH_U64 -static inline uint64 -pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U64 -static inline uint64 -pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - return pg_atomic_fetch_add_u64_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64 -static inline void -pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - (void) pg_atomic_exchange_u64_impl(ptr, val); -} -#endif diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 432509277c9..315014c96ad 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3861,6 +3861,8 @@ pendingPosition pending_label pgParameterStatus pg_atomic_flag +pg_atomic_uint8 +pg_atomic_uint16 pg_atomic_uint32 pg_atomic_uint64 pg_be_sasl_mech -- 2.50.1 (Apple Git-155)