From 44374af7026642f2cffa28dccea71f2d370a3ce1 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Wed, 29 Nov 2023 15:00:58 -0600
Subject: [PATCH v1 1/1] optimized atomic exchange

---
 config/c-compiler.m4                      | 34 ++++++++++-
 configure                                 | 72 +++++++++++++++++++++--
 configure.ac                              |  2 +
 meson.build                               | 12 ++++
 src/include/pg_config.h.in                |  6 ++
 src/include/port/atomics/arch-x86.h       | 28 +++++++++
 src/include/port/atomics/generic-gcc.h    | 28 +++++++++
 src/include/port/atomics/generic-msvc.h   | 18 ++++++
 src/include/port/atomics/generic-sunpro.h | 14 +++++
 src/tools/msvc/Solution.pm                |  2 +
 10 files changed, 210 insertions(+), 6 deletions(-)

diff --git a/config/c-compiler.m4 b/config/c-compiler.m4
index 5db02b2ab7..440c90d411 100644
--- a/config/c-compiler.m4
+++ b/config/c-compiler.m4
@@ -571,7 +571,7 @@ fi])# PGAC_HAVE_GCC__SYNC_INT64_CAS
 # Check if the C compiler understands __atomic_compare_exchange_n() for 32bit
 # types, and define HAVE_GCC__ATOMIC_INT32_CAS if so.
 AC_DEFUN([PGAC_HAVE_GCC__ATOMIC_INT32_CAS],
-[AC_CACHE_CHECK(for builtin __atomic int32 atomic operations, pgac_cv_gcc_atomic_int32_cas,
+[AC_CACHE_CHECK(for builtin __atomic int32 atomic compare/exchange, pgac_cv_gcc_atomic_int32_cas,
 [AC_LINK_IFELSE([AC_LANG_PROGRAM([],
   [int val = 0;
    int expect = 0;
@@ -587,7 +587,7 @@ fi])# PGAC_HAVE_GCC__ATOMIC_INT32_CAS
 # Check if the C compiler understands __atomic_compare_exchange_n() for 64bit
 # types, and define HAVE_GCC__ATOMIC_INT64_CAS if so.
 AC_DEFUN([PGAC_HAVE_GCC__ATOMIC_INT64_CAS],
-[AC_CACHE_CHECK(for builtin __atomic int64 atomic operations, pgac_cv_gcc_atomic_int64_cas,
+[AC_CACHE_CHECK(for builtin __atomic int64 atomic compare/exchange, pgac_cv_gcc_atomic_int64_cas,
 [AC_LINK_IFELSE([AC_LANG_PROGRAM([],
   [PG_INT64_TYPE val = 0;
    PG_INT64_TYPE expect = 0;
@@ -598,6 +598,36 @@ if test x"$pgac_cv_gcc_atomic_int64_cas" = x"yes"; then
   AC_DEFINE(HAVE_GCC__ATOMIC_INT64_CAS, 1, [Define to 1 if you have __atomic_compare_exchange_n(int64 *, int64 *, int64).])
 fi])# PGAC_HAVE_GCC__ATOMIC_INT64_CAS
 
+# PGAC_HAVE_GCC__ATOMIC_INT32_XCHG
+# --------------------------------
+# Check if the C compiler understands __atomic_exchange_n() for 32bit types,
+# and define HAVE_GCC__ATOMIC_INT32_XCHG if so.
+AC_DEFUN([PGAC_HAVE_GCC__ATOMIC_INT32_XCHG],
+[AC_CACHE_CHECK(for builtin __atomic int32 atomic exchange, pgac_cv_gcc_atomic_int32_xchg,
+[AC_LINK_IFELSE([AC_LANG_PROGRAM([],
+  [int val = 0;
+   __atomic_exchange_n(&val, 37, __ATOMIC_SEQ_CST);])],
+  [pgac_cv_gcc_atomic_int32_xchg="yes"],
+  [pgac_cv_gcc_atomic_int32_xchg="no"])])
+if test x"$pgac_cv_gcc_atomic_int32_xchg" = x"yes"; then
+  AC_DEFINE(HAVE_GCC__ATOMIC_INT32_XCHG, 1, [Define to 1 if you have __atomic_exchange_n(int *, int).])
+fi])# PGAC_HAVE_GCC__ATOMIC_INT32_XCHG
+
+# PGAC_HAVE_GCC__ATOMIC_INT64_XCHG
+# --------------------------------
+# Check if the C compiler understands __atomic_exchange_n() for 64bit types,
+# and define HAVE_GCC__ATOMIC_INT64_XCHG if so.
+AC_DEFUN([PGAC_HAVE_GCC__ATOMIC_INT64_XCHG],
+[AC_CACHE_CHECK(for builtin __atomic int64 atomic exchange, pgac_cv_gcc_atomic_int64_xchg,
+[AC_LINK_IFELSE([AC_LANG_PROGRAM([],
+  [PG_INT64_TYPE val = 0;
+   __atomic_exchange_n(&val, (PG_INT64_TYPE) 37, __ATOMIC_SEQ_CST);])],
+  [pgac_cv_gcc_atomic_int64_xchg="yes"],
+  [pgac_cv_gcc_atomic_int64_xchg="no"])])
+if test x"$pgac_cv_gcc_atomic_int64_xchg" = x"yes"; then
+  AC_DEFINE(HAVE_GCC__ATOMIC_INT64_XCHG, 1, [Define to 1 if you have __atomic_exchange_n(int64 *, int64).])
+fi])# PGAC_HAVE_GCC__ATOMIC_INT64_XCHG
+
 # PGAC_SSE42_CRC32_INTRINSICS
 # ---------------------------
 # Check if the compiler supports the x86 CRC instructions added in SSE 4.2,
diff --git a/configure b/configure
index 217704e9ca..d487e09b6e 100755
--- a/configure
+++ b/configure
@@ -17600,8 +17600,8 @@ if test x"$pgac_cv_gcc_sync_int64_cas" = x"yes"; then
 $as_echo "#define HAVE_GCC__SYNC_INT64_CAS 1" >>confdefs.h
 
 fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int32 atomic operations" >&5
-$as_echo_n "checking for builtin __atomic int32 atomic operations... " >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int32 atomic compare/exchange" >&5
+$as_echo_n "checking for builtin __atomic int32 atomic compare/exchange... " >&6; }
 if ${pgac_cv_gcc_atomic_int32_cas+:} false; then :
   $as_echo_n "(cached) " >&6
 else
@@ -17633,8 +17633,8 @@ if test x"$pgac_cv_gcc_atomic_int32_cas" = x"yes"; then
 $as_echo "#define HAVE_GCC__ATOMIC_INT32_CAS 1" >>confdefs.h
 
 fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int64 atomic operations" >&5
-$as_echo_n "checking for builtin __atomic int64 atomic operations... " >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int64 atomic compare/exchange" >&5
+$as_echo_n "checking for builtin __atomic int64 atomic compare/exchange... " >&6; }
 if ${pgac_cv_gcc_atomic_int64_cas+:} false; then :
   $as_echo_n "(cached) " >&6
 else
@@ -17665,6 +17665,70 @@ if test x"$pgac_cv_gcc_atomic_int64_cas" = x"yes"; then
 
 $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h
 
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int32 atomic exchange" >&5
+$as_echo_n "checking for builtin __atomic int32 atomic exchange... " >&6; }
+if ${pgac_cv_gcc_atomic_int32_xchg+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+int val = 0;
+   __atomic_exchange_n(&val, 37, __ATOMIC_SEQ_CST);
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  pgac_cv_gcc_atomic_int32_xchg="yes"
+else
+  pgac_cv_gcc_atomic_int32_xchg="no"
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_gcc_atomic_int32_xchg" >&5
+$as_echo "$pgac_cv_gcc_atomic_int32_xchg" >&6; }
+if test x"$pgac_cv_gcc_atomic_int32_xchg" = x"yes"; then
+
+$as_echo "#define HAVE_GCC__ATOMIC_INT32_XCHG 1" >>confdefs.h
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int64 atomic exchange" >&5
+$as_echo_n "checking for builtin __atomic int64 atomic exchange... " >&6; }
+if ${pgac_cv_gcc_atomic_int64_xchg+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+PG_INT64_TYPE val = 0;
+   __atomic_exchange_n(&val, (PG_INT64_TYPE) 37, __ATOMIC_SEQ_CST);
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  pgac_cv_gcc_atomic_int64_xchg="yes"
+else
+  pgac_cv_gcc_atomic_int64_xchg="no"
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_gcc_atomic_int64_xchg" >&5
+$as_echo "$pgac_cv_gcc_atomic_int64_xchg" >&6; }
+if test x"$pgac_cv_gcc_atomic_int64_xchg" = x"yes"; then
+
+$as_echo "#define HAVE_GCC__ATOMIC_INT64_XCHG 1" >>confdefs.h
+
 fi
 
 
diff --git a/configure.ac b/configure.ac
index e49de9e4f0..2234bda39a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2046,6 +2046,8 @@ PGAC_HAVE_GCC__SYNC_INT32_CAS
 PGAC_HAVE_GCC__SYNC_INT64_CAS
 PGAC_HAVE_GCC__ATOMIC_INT32_CAS
 PGAC_HAVE_GCC__ATOMIC_INT64_CAS
+PGAC_HAVE_GCC__ATOMIC_INT32_XCHG
+PGAC_HAVE_GCC__ATOMIC_INT64_XCHG
 
 
 # Check for x86 cpuid instruction
diff --git a/meson.build b/meson.build
index 0095fb183a..9e73b6eca2 100644
--- a/meson.build
+++ b/meson.build
@@ -1957,12 +1957,24 @@ int val = 0;
 int expect = 0;
 __atomic_compare_exchange_n(&val, &expect, 37, 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED);'''},
 
+    {'name': 'HAVE_GCC__ATOMIC_INT32_XCHG',
+     'desc': '__atomic_exchange_n(int32)',
+     'test': '''
+int val = 0;
+__atomic_exchange_n(&val, 37, __ATOMIC_SEQ_CST);'''},
+
     {'name': 'HAVE_GCC__ATOMIC_INT64_CAS',
      'desc': ' __atomic_compare_exchange_n(int64)',
      'test': '''
 INT64 val = 0;
 INT64 expect = 0;
 __atomic_compare_exchange_n(&val, &expect, 37, 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED);'''},
+
+    {'name': 'HAVE_GCC__ATOMIC_INT64_XCHG',
+     'desc': '__atomic_exchange_n(int64)',
+     'test': '''
+INT64 val = 0;
+__atomic_exchange_n(&val, (INT64) 37, __ATOMIC_SEQ_CST);'''},
   ]
 
   foreach check : atomic_checks
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 5f16918243..950c32a1dc 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -154,10 +154,16 @@
 /* Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int). */
 #undef HAVE_GCC__ATOMIC_INT32_CAS
 
+/* Define to 1 if you have __atomic_exchange_n(int *, int). */
+#undef HAVE_GCC__ATOMIC_INT32_XCHG
+
 /* Define to 1 if you have __atomic_compare_exchange_n(int64 *, int64 *,
    int64). */
 #undef HAVE_GCC__ATOMIC_INT64_CAS
 
+/* Define to 1 if you have __atomic_exchange_n(int64 *, int64). */
+#undef HAVE_GCC__ATOMIC_INT64_XCHG
+
 /* Define to 1 if you have __sync_lock_test_and_set(char *) and friends. */
 #undef HAVE_GCC__SYNC_CHAR_TAS
 
diff --git a/src/include/port/atomics/arch-x86.h b/src/include/port/atomics/arch-x86.h
index bb84b9bad8..8fce96228b 100644
--- a/src/include/port/atomics/arch-x86.h
+++ b/src/include/port/atomics/arch-x86.h
@@ -184,6 +184,20 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
 	return (bool) ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U32
+static inline uint32
+pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
+{
+	uint32 res;
+	__asm__ __volatile__(
+		"	lock				\n"
+		"	xchgl		%0,%1	\n"
+:		"=q" (res), "=m" (ptr->value)
+:		"0"(newval), "m" (ptr->value)
+:		"memory");
+	return res;
+}
+
 #define PG_HAVE_ATOMIC_FETCH_ADD_U32
 static inline uint32
 pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
@@ -221,6 +235,20 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
 	return (bool) ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U64
+static inline uint64
+pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
+{
+	uint64 res;
+	__asm__ __volatile__(
+		"   lock				\n"
+		"   xchgq		%0,%1	\n"
+:		"=q" (res), "=m" (ptr->value)
+:		"0"(newval), "m" (ptr->value)
+:		"memory");
+	return res;
+}
+
 #define PG_HAVE_ATOMIC_FETCH_ADD_U64
 static inline uint64
 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h
index da04e9f0dc..c35bbdc399 100644
--- a/src/include/port/atomics/generic-gcc.h
+++ b/src/include/port/atomics/generic-gcc.h
@@ -176,6 +176,20 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
 }
 #endif
 
+/*
+ * __sync_lock_test_and_set() only supports setting the value to 1 on some
+ * platforms, so we only provide an __atomic implementation for
+ * pg_atomic_exchange.
+ */
+#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_XCHG)
+#define PG_HAVE_ATOMIC_EXCHANGE_U32
+static inline uint32
+pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
+{
+	return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
+}
+#endif
+
 /* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */
 
 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
@@ -243,6 +257,20 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
 }
 #endif
 
+/*
+ * __sync_lock_test_and_set() only supports setting the value to 1 on some
+ * platforms, so we only provide an __atomic implementation for
+ * pg_atomic_exchange.
+ */
+#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_XCHG)
+#define PG_HAVE_ATOMIC_EXCHANGE_U64
+static inline uint64
+pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
+{
+	return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
+}
+#endif
+
 /* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */
 
 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h
index 8835f4ceea..7566d6f937 100644
--- a/src/include/port/atomics/generic-msvc.h
+++ b/src/include/port/atomics/generic-msvc.h
@@ -58,6 +58,13 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
 	return ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U32
+static inline uint32
+pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
+{
+	return InterlockedExchange(&ptr->value, newval);
+}
+
 #define PG_HAVE_ATOMIC_FETCH_ADD_U32
 static inline uint32
 pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
@@ -88,6 +95,16 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
 
 /* Only implemented on 64bit builds */
 #ifdef _WIN64
+
+#pragma intrinsic(_InterlockedExchange64)
+
+#define PG_HAVE_ATOMIC_EXCHANGE_U64
+static inline uint64
+pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
+{
+	return _InterlockedExchange64(&ptr->value, newval);
+}
+
 #pragma intrinsic(_InterlockedExchangeAdd64)
 
 #define PG_HAVE_ATOMIC_FETCH_ADD_U64
@@ -96,6 +113,7 @@ pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
 {
 	return _InterlockedExchangeAdd64(&ptr->value, add_);
 }
+
 #endif /* _WIN64 */
 
 #endif /* HAVE_ATOMICS */
diff --git a/src/include/port/atomics/generic-sunpro.h b/src/include/port/atomics/generic-sunpro.h
index 30f7d8b536..5d41b73de0 100644
--- a/src/include/port/atomics/generic-sunpro.h
+++ b/src/include/port/atomics/generic-sunpro.h
@@ -87,6 +87,13 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
 	return ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U32
+static inline uint32
+pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
+{
+	return atomic_swap_32(&ptr->value, newval);
+}
+
 #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
 static inline bool
 pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
@@ -101,6 +108,13 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
 	return ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U64
+static inline uint64
+pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
+{
+	return atomic_swap_64(&ptr->value, newval);
+}
+
 #endif /* HAVE_ATOMIC_H */
 
 #endif /* defined(HAVE_ATOMICS) */
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a980b51f5d..2d824b201e 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -249,7 +249,9 @@ sub GenerateFiles
 		HAVE_EXPLICIT_BZERO => undef,
 		HAVE_FSEEKO => 1,
 		HAVE_GCC__ATOMIC_INT32_CAS => undef,
+		HAVE_GCC__ATOMIC_INT32_XCHG => undef,
 		HAVE_GCC__ATOMIC_INT64_CAS => undef,
+		HAVE_GCC__ATOMIC_INT64_XCHG => undef,
 		HAVE_GCC__SYNC_CHAR_TAS => undef,
 		HAVE_GCC__SYNC_INT32_CAS => undef,
 		HAVE_GCC__SYNC_INT32_TAS => undef,
-- 
2.25.1

