From b969c5b6f7b6a991e5fc471bb986ac46de15cde4 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Fri, 20 Mar 2026 11:11:24 -0400 Subject: [PATCH v3 2/5] Redefine port/atomics.h on top of * Redirect types and functions to standard C/C++ * Use instead for C++11 extensions (as formalized by C++23) * Add pg_atomic_uint8, pg_atomic_uint16 and pg_atomic(T) * Provide generic pg_atomic_read(), pg_atomic_write() * Assume that most basic operations will be called by standardized generics in future code, eg atomic_fetch_add() * Forward all the traditional _32, _64 function names for existing code * Reimplement pg_atomic_flag with pg_atomic(uint8) to allow relaxed load (impossible with standard atomic_flag, and pg_atomic(bool) doesn't seem to be portable enough) * Let the compiler/runtime emulate 64 bit atomics with locks for armv7 * The per-compiler headers now just define pg_compiler_barrier_impl() * The per-arch headers now optionally define pg_{memory,read,write}_barrier_impl() but only if they think they can do something better than compilers (to investigate...) * Drop the secondary copy of the ancient spinlock ops from atomics * Drop arch-{arm,ppc}.h, fallback.h and generic.h WIP! TODO: Need to research generated code on all architectures and see if anything important is lost. --- src/include/c.h | 1 + src/include/port/atomics.h | 638 +++++++----------------- src/include/port/atomics/arch-arm.h | 32 -- src/include/port/atomics/arch-ppc.h | 256 ---------- src/include/port/atomics/arch-x86.h | 205 -------- src/include/port/atomics/fallback.h | 42 -- src/include/port/atomics/generic-gcc.h | 306 +----------- src/include/port/atomics/generic-msvc.h | 98 +--- src/include/port/atomics/generic.h | 436 ---------------- src/tools/pgindent/typedefs.list | 2 + 10 files changed, 173 insertions(+), 1843 deletions(-) delete mode 100644 src/include/port/atomics/arch-arm.h delete mode 100644 src/include/port/atomics/arch-ppc.h delete mode 100644 src/include/port/atomics/fallback.h delete mode 100644 src/include/port/atomics/generic.h diff --git a/src/include/c.h b/src/include/c.h index fd6b093bb3a..f4b8d6e2cd0 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -61,6 +61,7 @@ /* System header files that should be available everywhere in Postgres */ #include #include +#include #include #include #include diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index d8b1d20fe60..c6e3982c7b6 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -3,24 +3,33 @@ * atomics.h * Atomic operations. * - * Hardware and compiler dependent functions for manipulating memory - * atomically and dealing with cache coherency. Used to implement locking - * facilities and lockless algorithms/data structures. + * These interfaces are for manipulating memory atomically and dealing with + * cache coherency. They can be used to implement locking facilities and + * lockless algorithms/data structures. * - * To bring up postgres on a platform/compiler at the very least - * implementations for the following operations should be provided: - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() - * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. + * This is mostly a renaming of , with some differences: * - * There exist generic, hardware independent, implementations for several - * compilers which might be sufficient, although possibly not optimal, for a - * new platform. If no such generic implementation is available spinlocks will - * be used to implement the 64-bit parts of the API. + * * read/write barriers (weaker than acquire/release) + * * they affect even non-atomic access without dependency on atomic access + * * pg_atomic_flag semantics don't allow mapping to atomic_flag * - * Implement _u64 atomics if and only if your platform can use them - * efficiently (and obviously correctly). + * PostgreSQL atomic type width assumptions: + * + * 8, 16: Required to be lock-free (even though implemented with wider + * instructions by the compiler on some RISC-V systems). + * + * 32: Required to be lock-free. No known modern system lacks them. + * + * 64: These can reasonably be expected to be lock-free and fast on + * all modern-ish systems. For one known low-end system they are + * emulated by the compiler/runtime with locks (armv7). + * + * In all cases values must be well aligned or undefined behavior results. + * + * To bring up postgres on a compiler, provide: + * * pg_compiler_barrier() + * Optionally also: + * * pg_memory_barrier(), pg_write_barrier(), pg_read_barrier() * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -38,48 +47,66 @@ #ifndef ATOMICS_H #define ATOMICS_H -#ifdef FRONTEND -#error "atomics.h may not be included from frontend code" +#define INSIDE_ATOMICS_H + +#if defined(__cplusplus) && __cplusplus < 202302L +/* C++11 can't include C's . Use approach from C++23 33.5.12. */ +extern "C++" +{ +#include +#define pg_atomic(T) std::atomic + using std::memory_order_relaxed; + using std::memory_order_acquire; + using std::memory_order_release; + using std::memory_order_acq_rel; + using std::memory_order_seq_cst; +} +#else +#include +#define pg_atomic(T) _Atomic(T) #endif -#define INSIDE_ATOMICS_H +/* + * Visual Studio 2022 doesn't seem to define these in yet, but + * does define them to 2 in . They must hold in C too though, since + * they interoperate. + */ +#if !defined(_MSC_VER) || defined(__cplusplus) +/* Assumptions about lock-free access. */ +StaticAssertDecl(ATOMIC_CHAR_LOCK_FREE >= 2, "need lock-free 8-bit atomics"); +StaticAssertDecl(ATOMIC_SHORT_LOCK_FREE >= 2, "need lock-free 16-bit atomics"); +StaticAssertDecl(ATOMIC_INT_LOCK_FREE >= 2, "need lock-free 32-bit atomics"); +#endif + +/* Can't use standard atomic_flag due to extended semantics, see below. */ +typedef pg_atomic(uint8) +pg_atomic_flag; -#include +/* Common supported atomic integer types. */ +typedef pg_atomic(uint8) +pg_atomic_uint8; +typedef pg_atomic(uint16) +pg_atomic_uint16; +typedef pg_atomic(uint32) +pg_atomic_uint32; +typedef pg_atomic(uint64) +pg_atomic_uint64; /* * First a set of architecture specific files is included. * - * These files can provide the full set of atomics or can do pretty much - * nothing if all the compilers commonly used on these platforms provide - * usable generics. - * - * Don't add an inline assembly of the actual atomic operations if all the - * common implementations of your platform provide intrinsics. Intrinsics are - * much easier to understand and potentially support more architectures. - * * It will often make sense to define memory barrier semantics here, since * e.g. generic compiler intrinsics for x86 memory barriers can't know that * postgres doesn't need x86 read/write barriers do anything more than a * compiler barrier. - * */ -#if defined(__arm__) || defined(__arm) || defined(__aarch64__) -#include "port/atomics/arch-arm.h" -#elif defined(__i386__) || defined(__i386) || defined(__x86_64__) +#if defined(__i386__) || defined(__i386) || defined(__x86_64__) #include "port/atomics/arch-x86.h" -#elif defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) -#include "port/atomics/arch-ppc.h" #endif /* * Compiler specific, but architecture independent implementations. - * - * Provide architecture independent implementations of the atomic - * facilities. At the very least compiler barriers should be provided, but a - * full implementation of - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * using compiler intrinsics are a good idea. + * This defines compiler barriers. */ /* * gcc or compatible, including clang and icc. @@ -93,29 +120,9 @@ #endif /* Fail if we couldn't find implementations of required facilities. */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#error "could not find an implementation of pg_atomic_uint32" -#endif #if !defined(pg_compiler_barrier_impl) #error "could not find an implementation of pg_compiler_barrier" #endif -#if !defined(pg_memory_barrier_impl) -#error "could not find an implementation of pg_memory_barrier_impl" -#endif - - -/* - * Provide a spinlock-based implementation of the 64 bit variants, if - * necessary. - */ -#include "port/atomics/fallback.h" - -/* - * Provide additional operations using supported infrastructure. These are - * expected to be efficient if the underlying atomic operations are efficient. - */ -#include "port/atomics/generic.h" - /* * pg_compiler_barrier - prevent the compiler from moving code across @@ -125,6 +132,10 @@ * reorder loads or stores to main memory around the barrier. However, the * CPU may still reorder loads or stores at runtime, if the architecture's * memory model permits this. + * + * Unlike standard atomic_signal_fence(), pg_compiler_barrier() prevents + * reordering of non-atomic, non-volatile accesses, so a compiler-specific + * definition is required. */ #define pg_compiler_barrier() pg_compiler_barrier_impl() @@ -137,8 +148,20 @@ * loads and stores are totally ordered (which is not the case on most * architectures) this requires issuing some sort of memory fencing * instruction. + * + * Unlike standard atomic_thread_fence(), pg_{read,write}_barrier() affects + * non-atomic, non-volatile accesses. */ -#define pg_memory_barrier() pg_memory_barrier_impl() +static inline void +pg_memory_barrier(void) +{ +#ifdef pg_memory_barrier_impl + pg_memory_barrier_impl(); +#else + pg_compiler_barrier(); + atomic_thread_fence(memory_order_seq_cst); +#endif +} /* * pg_(read|write)_barrier - prevent the CPU from reordering memory access @@ -150,439 +173,116 @@ * are thus weaker than a full memory barrier, but stronger than a compiler * barrier. In practice, on machines with strong memory ordering, read and * write barriers may require nothing more than a compiler barrier. - */ -#define pg_read_barrier() pg_read_barrier_impl() -#define pg_write_barrier() pg_write_barrier_impl() - -/* - * Spinloop delay - Allow CPU to relax in busy loops - */ -#define pg_spin_delay() pg_spin_delay_impl() - -/* - * pg_atomic_init_flag - initialize atomic flag. - * - * No barrier semantics. - */ -static inline void -pg_atomic_init_flag(volatile pg_atomic_flag *ptr) -{ - pg_atomic_init_flag_impl(ptr); -} - -/* - * pg_atomic_test_set_flag - TAS() - * - * Returns true if the flag has successfully been set, false otherwise. - * - * Acquire (including read barrier) semantics. - */ -static inline bool -pg_atomic_test_set_flag(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_test_set_flag_impl(ptr); -} - -/* - * pg_atomic_unlocked_test_flag - Check if the lock is free - * - * Returns true if the flag currently is not set, false otherwise. * - * No barrier semantics. + * Unlike standard atomic_thread_fence(), pg_{read,write}_barrier() affects + * non-atomic, non-volatile accesses. The default implementation uses + * acquire/release fences, but these are strictly stronger than read/write. */ -static inline bool -pg_atomic_unlocked_test_flag(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_unlocked_test_flag_impl(ptr); -} -/* - * pg_atomic_clear_flag - release lock set by TAS() - * - * Release (including write barrier) semantics. - */ static inline void -pg_atomic_clear_flag(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} - - -/* - * pg_atomic_init_u32 - initialize atomic variable - * - * Has to be done before any concurrent usage.. - * - * No barrier semantics. - */ -static inline void -pg_atomic_init_u32(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - AssertPointerAlignment(ptr, 4); - - pg_atomic_init_u32_impl(ptr, val); -} - -/* - * pg_atomic_read_u32 - unlocked read from atomic variable. - * - * The read is guaranteed to return a value as it has been written by this or - * another process at some point in the past. There's however no cache - * coherency interaction guaranteeing the value hasn't since been written to - * again. - * - * No barrier semantics. - */ -static inline uint32 -pg_atomic_read_u32(volatile pg_atomic_uint32 *ptr) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_read_u32_impl(ptr); -} - -/* - * pg_atomic_read_membarrier_u32 - read with barrier semantics. - * - * This read is guaranteed to return the current value, provided that the value - * is only ever updated via operations with barrier semantics, such as - * pg_atomic_compare_exchange_u32() and pg_atomic_write_membarrier_u32(). - * While this may be less performant than pg_atomic_read_u32(), it may be - * easier to reason about correctness with this function in less performance- - * sensitive code. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_read_membarrier_u32(volatile pg_atomic_uint32 *ptr) +pg_read_barrier(void) { - AssertPointerAlignment(ptr, 4); - - return pg_atomic_read_membarrier_u32_impl(ptr); -} - -/* - * pg_atomic_write_u32 - write to atomic variable. - * - * The write is guaranteed to succeed as a whole, i.e. it's not possible to - * observe a partial write for any reader. Note that this correctly interacts - * with pg_atomic_compare_exchange_u32, in contrast to - * pg_atomic_unlocked_write_u32(). - * - * No barrier semantics. - */ -static inline void -pg_atomic_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - AssertPointerAlignment(ptr, 4); - - pg_atomic_write_u32_impl(ptr, val); -} - -/* - * pg_atomic_unlocked_write_u32 - unlocked write to atomic variable. - * - * Write to an atomic variable, without atomicity guarantees. I.e. it is not - * guaranteed that a concurrent reader will not see a torn value, nor is this - * guaranteed to correctly interact with concurrent read-modify-write - * operations like pg_atomic_compare_exchange_u32. This should only be used - * in cases where minor performance regressions due to atomic operations are - * unacceptable and where exclusive access is guaranteed via some external - * means. - * - * No barrier semantics. - */ -static inline void -pg_atomic_unlocked_write_u32(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - AssertPointerAlignment(ptr, 4); - - pg_atomic_unlocked_write_u32_impl(ptr, val); +#ifdef pg_read_barrier_impl + pg_read_barrier_impl(); +#else + pg_compiler_barrier(); + atomic_thread_fence(memory_order_acquire); +#endif } -/* - * pg_atomic_write_membarrier_u32 - write with barrier semantics. - * - * The write is guaranteed to succeed as a whole, i.e., it's not possible to - * observe a partial write for any reader. Note that this correctly interacts - * with both pg_atomic_compare_exchange_u32() and - * pg_atomic_read_membarrier_u32(). While this may be less performant than - * pg_atomic_write_u32(), it may be easier to reason about correctness with - * this function in less performance-sensitive code. - * - * Full barrier semantics. - */ static inline void -pg_atomic_write_membarrier_u32(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - AssertPointerAlignment(ptr, 4); - - pg_atomic_write_membarrier_u32_impl(ptr, val); -} - -/* - * pg_atomic_exchange_u32 - exchange newval with current value - * - * Returns the old value of 'ptr' before the swap. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_exchange_u32(volatile pg_atomic_uint32 *ptr, uint32 newval) +pg_write_barrier(void) { - AssertPointerAlignment(ptr, 4); - - return pg_atomic_exchange_u32_impl(ptr, newval); +#ifdef pg_write_barrier_impl + pg_write_barrier_impl(); +#else + pg_compiler_barrier(); + atomic_thread_fence(memory_order_release); +#endif } /* - * pg_atomic_compare_exchange_u32 - CAS operation + * Operations corresponding to standard atomic_flag. We have to use an integer + * instead of mapping directly to the standard names, to support our relaxed + * check function. * - * Atomically compare the current value of ptr with *expected and store newval - * iff ptr and *expected have the same value. The current value of *ptr will - * always be stored in *expected. - * - * Return true if values have been exchanged, false otherwise. - * - * Full barrier semantics. + * Acquire and release fences, which are respectively stronger than read and + * write barriers. */ -static inline bool -pg_atomic_compare_exchange_u32(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - AssertPointerAlignment(ptr, 4); - AssertPointerAlignment(expected, 4); - - return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval); -} +#define pg_atomic_init_flag(p) atomic_init((p), 1) +#define pg_atomic_test_set_flag(p) \ + atomic_fetch_and_explicit((p), 0, memory_order_acquire) +#define pg_atomic_clear_flag(p) \ + atomic_store_explicit((p), 1, memory_order_release) +#define pg_atomic_unlocked_test_flag(p) \ + atomic_load_explicit((p), memory_order_relaxed) /* - * pg_atomic_fetch_add_u32 - atomically add to variable - * - * Returns the value of ptr before the arithmetic operation. - * - * Full barrier semantics. + * Local convention for load/store, relaxed AKA no barrier. These are the only + * "generic" functions provided with pg_ prefixes. It seems pointless to + * rename everything in , but these two are PostgreSQL's + * established way of representing relaxed access, and a lot shorter. */ -static inline uint32 -pg_atomic_fetch_add_u32(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_fetch_add_u32_impl(ptr, add_); -} +#define pg_atomic_read(p) \ + atomic_load_explicit((p), memory_order_relaxed) +#define pg_atomic_write(p, v) \ + atomic_store_explicit((p), (v), memory_order_relaxed) /* - * pg_atomic_fetch_sub_u32 - atomically subtract from variable + * Backward-compatible type-specific function names. Only the historical _u32 + * and _64 names are provided. New code and code using other atomic types + * should use the generic functions from the standard directly, and + * pg_atomic_{read,write}() for relaxed access. * - * Returns the value of ptr before the arithmetic operation. Note that sub_ - * may not be INT_MIN due to platform limitations. - * - * Full barrier semantics. + * All of these except pg_atomic_(unlocked_){read,write}_{32,64} have seq cst + * AKA full barrier semantics. */ -static inline uint32 -pg_atomic_fetch_sub_u32(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - AssertPointerAlignment(ptr, 4); - Assert(sub_ != INT_MIN); - return pg_atomic_fetch_sub_u32_impl(ptr, sub_); -} +#define pg_atomic_init_u32 atomic_init +#define pg_atomic_init_u64 atomic_init +#define pg_atomic_read_u32 pg_atomic_read +#define pg_atomic_read_u64 pg_atomic_read +#define pg_atomic_write_u32 pg_atomic_write +#define pg_atomic_write_u64 pg_atomic_write +#define pg_atomic_unlocked_read_u32 pg_atomic_read +#define pg_atomic_unlocked_read_u64 pg_atomic_read +#define pg_atomic_unlocked_write_u32 pg_atomic_write +#define pg_atomic_unlocked_write_u64 pg_atomic_write +#define pg_atomic_read_membarrier_u32 atomic_load +#define pg_atomic_read_membarrier_u64 atomic_load +#define pg_atomic_write_membarrier_u32 atomic_store +#define pg_atomic_write_membarrier_u64 atomic_store +#define pg_atomic_exchange_u32 atomic_exchange +#define pg_atomic_exchange_u64 atomic_exchange +#define pg_atomic_compare_exchange_u32 atomic_compare_exchange_strong +#define pg_atomic_compare_exchange_u64 atomic_compare_exchange_strong +#define pg_atomic_fetch_add_u32 atomic_fetch_add +#define pg_atomic_fetch_add_u64 atomic_fetch_add +#define pg_atomic_fetch_sub_u32 atomic_fetch_sub +#define pg_atomic_fetch_sub_u64 atomic_fetch_sub +#define pg_atomic_fetch_and_u32 atomic_fetch_and +#define pg_atomic_fetch_and_u64 atomic_fetch_and +#define pg_atomic_fetch_or_u32 atomic_fetch_or +#define pg_atomic_fetch_or_u64 atomic_fetch_or /* - * pg_atomic_fetch_and_u32 - atomically bit-and and_ with variable - * - * Returns the value of ptr before the arithmetic operation. - * - * Full barrier semantics. + * The rest of this file defines non-fundamental convenience functions with no + * counterpart in . */ -static inline uint32 -pg_atomic_fetch_and_u32(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_fetch_and_u32_impl(ptr, and_); -} /* - * pg_atomic_fetch_or_u32 - atomically bit-or or_ with variable - * - * Returns the value of ptr before the arithmetic operation. - * - * Full barrier semantics. + * Fetch and add/subtract wrappers that return the new value instead of the old + * value. We need functions to avoid double evaluation of v. */ -static inline uint32 -pg_atomic_fetch_or_u32(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_fetch_or_u32_impl(ptr, or_); -} - -/* - * pg_atomic_add_fetch_u32 - atomically add to variable - * - * Returns the value of ptr after the arithmetic operation. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_add_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - AssertPointerAlignment(ptr, 4); - return pg_atomic_add_fetch_u32_impl(ptr, add_); -} - -/* - * pg_atomic_sub_fetch_u32 - atomically subtract from variable - * - * Returns the value of ptr after the arithmetic operation. Note that sub_ may - * not be INT_MIN due to platform limitations. - * - * Full barrier semantics. - */ -static inline uint32 -pg_atomic_sub_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - AssertPointerAlignment(ptr, 4); - Assert(sub_ != INT_MIN); - return pg_atomic_sub_fetch_u32_impl(ptr, sub_); -} - -/* ---- - * The 64 bit operations have the same semantics as their 32bit counterparts - * if they are available. Check the corresponding 32bit function for - * documentation. - * ---- - */ -static inline void -pg_atomic_init_u64(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * Can't necessarily enforce alignment - and don't need it - when using - * the spinlock based fallback implementation. Therefore only assert when - * not using it. - */ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - pg_atomic_init_u64_impl(ptr, val); -} - -static inline uint64 -pg_atomic_read_u64(volatile pg_atomic_uint64 *ptr) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_read_u64_impl(ptr); -} - -static inline uint64 -pg_atomic_read_membarrier_u64(volatile pg_atomic_uint64 *ptr) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_read_membarrier_u64_impl(ptr); -} - -static inline void -pg_atomic_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - pg_atomic_write_u64_impl(ptr, val); -} - -static inline void -pg_atomic_unlocked_write_u64(volatile pg_atomic_uint64 *ptr, uint64 val) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - - pg_atomic_unlocked_write_u64_impl(ptr, val); -} - -static inline void -pg_atomic_write_membarrier_u64(volatile pg_atomic_uint64 *ptr, uint64 val) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - pg_atomic_write_membarrier_u64_impl(ptr, val); -} - -static inline uint64 -pg_atomic_exchange_u64(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_exchange_u64_impl(ptr, newval); -} - -static inline bool -pg_atomic_compare_exchange_u64(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval); -} - -static inline uint64 -pg_atomic_fetch_add_u64(volatile pg_atomic_uint64 *ptr, int64 add_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_fetch_add_u64_impl(ptr, add_); -} - -static inline uint64 -pg_atomic_fetch_sub_u64(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - Assert(sub_ != PG_INT64_MIN); - return pg_atomic_fetch_sub_u64_impl(ptr, sub_); -} - -static inline uint64 -pg_atomic_fetch_and_u64(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_fetch_and_u64_impl(ptr, and_); -} - -static inline uint64 -pg_atomic_fetch_or_u64(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_fetch_or_u64_impl(ptr, or_); -} - -static inline uint64 -pg_atomic_add_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 add_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - return pg_atomic_add_fetch_u64_impl(ptr, add_); -} - -static inline uint64 -pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - Assert(sub_ != PG_INT64_MIN); - return pg_atomic_sub_fetch_u64_impl(ptr, sub_); +#define PG_ATOMIC_GEN_REVERSE_FETCH(size, name, op) \ +static inline uint##size \ +pg_atomic_##name##_fetch_u##size(volatile pg_atomic_uint##size *p, uint##size v) \ +{ \ + return atomic_fetch_##name(p, v) op v; \ } +PG_ATOMIC_GEN_REVERSE_FETCH(32, add, +); +PG_ATOMIC_GEN_REVERSE_FETCH(64, add, +); +PG_ATOMIC_GEN_REVERSE_FETCH(32, sub, -); +PG_ATOMIC_GEN_REVERSE_FETCH(64, sub, -); /* * Monotonically advance the given variable using only atomic operations until @@ -596,11 +296,7 @@ pg_atomic_monotonic_advance_u64(volatile pg_atomic_uint64 *ptr, uint64 target) { uint64 currval; -#ifndef PG_HAVE_ATOMIC_U64_SIMULATION - AssertPointerAlignment(ptr, 8); -#endif - - currval = pg_atomic_read_u64_impl(ptr); + currval = pg_atomic_read_u64(ptr); if (currval >= target) { pg_memory_barrier(); diff --git a/src/include/port/atomics/arch-arm.h b/src/include/port/atomics/arch-arm.h deleted file mode 100644 index 90280c7b751..00000000000 --- a/src/include/port/atomics/arch-arm.h +++ /dev/null @@ -1,32 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-arm.h - * Atomic operations considerations specific to ARM - * - * Portions Copyright (c) 2013-2026, PostgreSQL Global Development Group - * - * NOTES: - * - * src/include/port/atomics/arch-arm.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * 64 bit atomics on ARM32 are implemented using kernel fallbacks and thus - * might be slow, so disable entirely. On ARM64 that problem doesn't exist. - */ -#if !defined(__aarch64__) -#define PG_DISABLE_64_BIT_ATOMICS -#else -/* - * Architecture Reference Manual for ARMv8 states aligned read/write to/from - * general purpose register is atomic. - */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* __aarch64__ */ diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h deleted file mode 100644 index 65ced2c6d3a..00000000000 --- a/src/include/port/atomics/arch-ppc.h +++ /dev/null @@ -1,256 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-ppc.h - * Atomic operations considerations specific to PowerPC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-ppc.h - * - *------------------------------------------------------------------------- - */ - -#if defined(__GNUC__) - -/* - * lwsync orders loads with respect to each other, and similarly with stores. - * But a load can be performed before a subsequent store, so sync must be used - * for a full memory barrier. - */ -#define pg_memory_barrier_impl() __asm__ __volatile__ ("sync" : : : "memory") -#define pg_read_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#define pg_write_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* 64bit atomics are only supported in 64bit mode */ -#if SIZEOF_VOID_P >= 8 -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif - -/* - * This mimics gcc __atomic_compare_exchange_n(..., __ATOMIC_SEQ_CST), but - * code generation differs at the end. __atomic_compare_exchange_n(): - * 100: isync - * 104: mfcr r3 - * 108: rlwinm r3,r3,3,31,31 - * 10c: bne 120 <.eb+0x10> - * 110: clrldi r3,r3,63 - * 114: addi r1,r1,112 - * 118: blr - * 11c: nop - * 120: clrldi r3,r3,63 - * 124: stw r9,0(r4) - * 128: addi r1,r1,112 - * 12c: blr - * - * This: - * f0: isync - * f4: mfcr r9 - * f8: rldicl. r3,r9,35,63 - * fc: bne 104 <.eb> - * 100: stw r10,0(r4) - * 104: addi r1,r1,112 - * 108: blr - * - * This implementation may or may not have materially different performance. - * It's not exploiting the fact that cr0 still holds the relevant comparison - * bits, set during the __asm__. One could fix that by moving more code into - * the __asm__. (That would remove the freedom to eliminate dead stores when - * the caller ignores "expected", but few callers do.) - * - * Recognizing constant "newval" would be superfluous, because there's no - * immediate-operand version of stwcx. - */ -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - uint32 found; - uint32 condition_register; - bool ret; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int32) *expected <= PG_INT16_MAX && - (int32) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpwi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpw %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -/* - * This mirrors gcc __sync_fetch_and_add(). - * - * Like tas(), use constraint "=&b" to avoid allocating r0. - */ -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 _t; - uint32 res; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#ifdef PG_HAVE_ATOMIC_U64_SUPPORT - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - uint64 found; - uint32 condition_register; - bool ret; - - AssertPointerAlignment(expected, 8); - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/; s/cmpw/cmpd/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int64) *expected <= PG_INT16_MAX && - (int64) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpdi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpd %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 _t; - uint64 res; - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ - -/* per architecture manual doubleword accesses have single copy atomicity */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY diff --git a/src/include/port/atomics/arch-x86.h b/src/include/port/atomics/arch-x86.h index bd6f4f56ca2..bf4fd660ccb 100644 --- a/src/include/port/atomics/arch-x86.h +++ b/src/include/port/atomics/arch-x86.h @@ -3,10 +3,6 @@ * arch-x86.h * Atomic operations considerations specific to intel x86 * - * Note that we actually require a 486 upwards because the 386 doesn't have - * support for xadd and cmpxchg. Given that the 386 isn't supported anywhere - * anymore that's not much of a restriction luckily. - * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * @@ -43,204 +39,3 @@ #define pg_read_barrier_impl() pg_compiler_barrier_impl() #define pg_write_barrier_impl() pg_compiler_barrier_impl() - -/* - * Provide implementation for atomics using inline assembly on x86 gcc. It's - * nice to support older gcc's and the compare/exchange implementation here is - * actually more efficient than the * __sync variant. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - volatile char value; -} pg_atomic_flag; - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* - * It's too complicated to write inline asm for 64bit types on 32bit and the - * 486 can't do it anyway. - */ -#ifdef __x86_64__ -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - /* alignment guaranteed due to being on a 64bit platform */ - volatile uint64 value; -} pg_atomic_uint64; -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -#if !defined(PG_HAVE_SPIN_DELAY) -/* - * This sequence is equivalent to the PAUSE instruction ("rep" is - * ignored by old IA32 processors if the following instruction is - * not a string operation); the IA-32 Architecture Software - * Developer's Manual, Vol. 3, Section 7.7.2 describes why using - * PAUSE in the inner loop of a spin lock is necessary for good - * performance: - * - * The PAUSE instruction improves the performance of IA-32 - * processors supporting Hyper-Threading Technology when - * executing spin-wait loops and other routines where one - * thread is accessing a shared lock or semaphore in a tight - * polling loop. When executing a spin-wait loop, the - * processor can suffer a severe performance penalty when - * exiting the loop because it detects a possible memory order - * violation and flushes the core processor's pipeline. The - * PAUSE instruction provides a hint to the processor that the - * code sequence is a spin-wait loop. The processor uses this - * hint to avoid the memory order violation and prevent the - * pipeline flush. In addition, the PAUSE instruction - * de-pipelines the spin-wait loop to prevent it from - * consuming execution resources excessively. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#define PG_HAVE_SPIN_DELAY -static inline void -pg_spin_delay_impl(void) -{ - __asm__ __volatile__(" rep; nop \n"); -} -#elif defined(_MSC_VER) && defined(__x86_64__) -#define PG_HAVE_SPIN_DELAY -static __forceinline void -pg_spin_delay_impl(void) -{ - _mm_pause(); -} -#elif defined(_MSC_VER) -#define PG_HAVE_SPIN_DELAY -static __forceinline void -pg_spin_delay_impl(void) -{ - /* See comment for gcc code. Same code, MASM syntax */ - __asm rep nop; -} -#endif -#endif /* !defined(PG_HAVE_SPIN_DELAY) */ - - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - char _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(ptr->value) -: -: "memory"); - return _res == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* - * On a TSO architecture like x86 it's sufficient to use a compiler - * barrier to achieve release semantics. - */ - __asm__ __volatile__("" ::: "memory"); - ptr->value = 0; -} - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - char ret; - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgl %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 res; - __asm__ __volatile__( - " lock \n" - " xaddl %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#ifdef __x86_64__ - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - char ret; - - AssertPointerAlignment(expected, 8); - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgq %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 res; - __asm__ __volatile__( - " lock \n" - " xaddq %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -/* - * 8 byte reads / writes have single-copy atomicity on 32 bit x86 platforms - * since at least the 586. As well as on all x86-64 cpus. - */ -#if defined(__i568__) || defined(__i668__) || /* gcc i586+ */ \ - (defined(_M_IX86) && _M_IX86 >= 500) || /* msvc i586+ */ \ - defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) /* gcc, msvc */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* 8 byte single-copy atomicity */ diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h deleted file mode 100644 index f37225189fc..00000000000 --- a/src/include/port/atomics/fallback.h +++ /dev/null @@ -1,42 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fallback.h - * Fallback for platforms without 64 bit atomics support. Slower - * than native atomics support, but not unusably slow. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/fallback.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - - -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) - -#define PG_HAVE_ATOMIC_U64_SIMULATION - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - int sema; - volatile uint64 value; -} pg_atomic_uint64; - -#define PG_HAVE_ATOMIC_INIT_U64 -extern void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_); - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval); - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_); - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h index 5bfce82f687..af8b55725b0 100644 --- a/src/include/port/atomics/generic-gcc.h +++ b/src/include/port/atomics/generic-gcc.h @@ -1,19 +1,11 @@ /*------------------------------------------------------------------------- * * generic-gcc.h - * Atomic operations, implemented using gcc (or compatible) intrinsics. + * Compiler barriers for GCC (or compatible) * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * NOTES: - * - * Documentation: - * * Legacy __sync Built-in Functions for Atomic Memory Access - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html - * * Built-in functions for memory model aware atomic operations - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html - * * src/include/port/atomics/generic-gcc.h * *------------------------------------------------------------------------- @@ -28,299 +20,3 @@ * An empty asm block should be a sufficient compiler barrier. */ #define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory") - -/* - * If we're on GCC, we should be able to get a memory barrier - * out of this compiler built-in. But we prefer to rely on platform specific - * definitions where possible, and use this only as a fallback. - */ -#if !defined(pg_memory_barrier_impl) -# if defined(HAVE_GCC__ATOMIC_INT32_CAS) -# define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST) -# elif defined(__GNUC__) -# define pg_memory_barrier_impl() __sync_synchronize() -# endif -#endif /* !defined(pg_memory_barrier_impl) */ - -#if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* acquire semantics include read barrier semantics */ -# define pg_read_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_ACQUIRE); \ -} while (0) -#endif - -#if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* release semantics include write barrier semantics */ -# define pg_write_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_RELEASE); \ -} while (0) -#endif - - -/* generic gcc based atomic flag implementation */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \ - && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS)) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - /* - * If we have a choice, use int-width TAS, because that is more efficient - * and/or more reliably implemented on most non-Intel platforms. (Note - * that this code isn't used on x86[_64]; see arch-x86.h for that.) - */ -#ifdef HAVE_GCC__SYNC_INT32_TAS - volatile int value; -#else - volatile char value; -#endif -} pg_atomic_flag; - -#endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */ - -/* generic gcc based atomic uint32 implementation */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \ - && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS)) - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */ - -/* generic gcc based atomic uint64 implementation */ -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \ - && !defined(PG_DISABLE_64_BIT_ATOMICS) \ - && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS)) - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ - -#ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT - -#if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) - -#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* NB: only an acquire barrier, not a full one */ - /* some platform only support a 1 here */ - return __sync_lock_test_and_set(&ptr->value, 1) == 0; -} -#endif - -#endif /* defined(HAVE_GCC__SYNC_*_TAS) */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return ptr->value == 0; -} -#endif - -#ifndef PG_HAVE_ATOMIC_CLEAR_FLAG -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - __sync_lock_release(&ptr->value); -} -#endif - -#ifndef PG_HAVE_ATOMIC_INIT_FLAG -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} -#endif - -#endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */ - -/* prefer __atomic, it has a better API */ -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - /* FIXME: we can probably use a lower consistency model */ - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 32-bit __atomic_compare_exchange_n() implies - * the availability of 32-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - - -#if !defined(PG_DISABLE_64_BIT_ATOMICS) - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - AssertPointerAlignment(expected, 8); - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - - AssertPointerAlignment(expected, 8); - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 64-bit __atomic_compare_exchange_n() implies - * the availability of 64-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - -#endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */ diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h index 6d09cd0e043..7ca45ff2730 100644 --- a/src/include/port/atomics/generic-msvc.h +++ b/src/include/port/atomics/generic-msvc.h @@ -1,17 +1,11 @@ /*------------------------------------------------------------------------- * * generic-msvc.h - * Atomic operations support when using MSVC + * Compiler barriers when using MSVC * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * NOTES: - * - * Documentation: - * * Interlocked Variable Access - * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx - * * src/include/port/atomics/generic-msvc.h * *------------------------------------------------------------------------- @@ -24,92 +18,4 @@ #endif #pragma intrinsic(_ReadWriteBarrier) -#define pg_compiler_barrier_impl() _ReadWriteBarrier() - -#ifndef pg_memory_barrier_impl -#define pg_memory_barrier_impl() MemoryBarrier() -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = InterlockedCompareExchange(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return InterlockedExchange(&ptr->value, newval); -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return InterlockedExchangeAdd(&ptr->value, add_); -} - -/* - * The non-intrinsics versions are only available in vista upwards, so use the - * intrinsic version. Only supported on >486, but we require XP as a minimum - * baseline, which doesn't support the 486, so we don't need to add checks for - * that case. - */ -#pragma intrinsic(_InterlockedCompareExchange64) - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - current = _InterlockedCompareExchange64(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -/* Only implemented on 64bit builds */ -#ifdef _WIN64 - -#pragma intrinsic(_InterlockedExchange64) - -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return _InterlockedExchange64(&ptr->value, newval); -} - -#pragma intrinsic(_InterlockedExchangeAdd64) - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return _InterlockedExchangeAdd64(&ptr->value, add_); -} - -#endif /* _WIN64 */ +#define pg_compiler_barrier_impl() _ReadWriteBarrier() diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h deleted file mode 100644 index fd64b6cbd86..00000000000 --- a/src/include/port/atomics/generic.h +++ /dev/null @@ -1,436 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic.h - * Implement higher level operations based on some lower level atomic - * operations. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/generic.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - -/* - * If read or write barriers are undefined, we upgrade them to full memory - * barriers. - */ -#if !defined(pg_read_barrier_impl) -# define pg_read_barrier_impl pg_memory_barrier_impl -#endif -#if !defined(pg_write_barrier_impl) -# define pg_write_barrier_impl pg_memory_barrier_impl -#endif - -#ifndef PG_HAVE_SPIN_DELAY -#define PG_HAVE_SPIN_DELAY -#define pg_spin_delay_impl() ((void)0) -#endif - - -/* provide fallback */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef pg_atomic_uint32 pg_atomic_flag; -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U32 -#define PG_HAVE_ATOMIC_READ_U32 -static inline uint32 -pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return ptr->value; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U32 -#define PG_HAVE_ATOMIC_WRITE_U32 -static inline void -pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -static inline void -pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -/* - * provide fallback for test_and_set using atomic_exchange if available - */ -#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_exchange_u32_impl(ptr, 1) == 0; -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -/* - * provide fallback for test_and_set using atomic_compare_exchange if - * available. - */ -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - uint32 value = 0; - return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1); -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) -# error "No pg_atomic_test_and_set provided" -#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */ - - -#ifndef PG_HAVE_ATOMIC_INIT_U32 -#define PG_HAVE_ATOMIC_INIT_U32 -static inline void -pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_ADD_FETCH_U32 -static inline uint32 -pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) -#define PG_HAVE_ATOMIC_SUB_FETCH_U32 -static inline uint32 -pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U32 -static inline uint32 -pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return pg_atomic_fetch_add_u32_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32 -static inline void -pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - (void) pg_atomic_exchange_u32_impl(ptr, val); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U64 -#define PG_HAVE_ATOMIC_WRITE_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * On this platform aligned 64bit writes are guaranteed to be atomic, - * except if using the fallback implementation, where can't guarantee the - * required alignment. - */ - AssertPointerAlignment(ptr, 8); - ptr->value = val; -} - -#else - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * 64 bit writes aren't safe on all platforms. In the generic - * implementation implement them as an atomic exchange. - */ - pg_atomic_exchange_u64_impl(ptr, val); -} - -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_WRITE_U64 */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -static inline void -pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U64 -#define PG_HAVE_ATOMIC_READ_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - /* - * On this platform aligned 64-bit reads are guaranteed to be atomic. - */ - AssertPointerAlignment(ptr, 8); - return ptr->value; -} - -#else - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - uint64 old = 0; - - /* - * 64-bit reads aren't atomic on all platforms. In the generic - * implementation implement them as a compare/exchange with 0. That'll - * fail or succeed, but always return the old value. Possibly might store - * a 0, but only if the previous value also was a 0 - i.e. harmless. - */ - pg_atomic_compare_exchange_u64_impl(ptr, &old, 0); - - return old; -} -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_READ_U64 */ - -#ifndef PG_HAVE_ATOMIC_INIT_U64 -#define PG_HAVE_ATOMIC_INIT_U64 -static inline void -pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_ADD_FETCH_U64 -static inline uint64 -pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) -#define PG_HAVE_ATOMIC_SUB_FETCH_U64 -static inline uint64 -pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U64 -static inline uint64 -pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - return pg_atomic_fetch_add_u64_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64 -static inline void -pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - (void) pg_atomic_exchange_u64_impl(ptr, val); -} -#endif diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 0042c33fa66..10fbb6726bc 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3947,6 +3947,8 @@ pending_label pgParameterStatus pgoff_t pg_atomic_flag +pg_atomic_uint8 +pg_atomic_uint16 pg_atomic_uint32 pg_atomic_uint64 pg_be_sasl_mech -- 2.51.2