From 57256412ce11b006ad383fc689a0fd28716632e0 Mon Sep 17 00:00:00 2001 From: Paul Amonson Date: Wed, 13 Mar 2024 11:56:44 -0700 Subject: [PATCH 1/2] [Refactor] POPCNT code refactored for future acceleration of the function. Signed-off-by: Paul Amonson --- src/port/Makefile | 2 + src/port/meson.build | 2 + src/port/pg_bitutils.c | 124 +----------------------------- src/port/pg_popcnt_choose.c | 97 +++++++++++++++++++++++ src/port/pg_popcnt_x86_64_accel.c | 55 +++++++++++++ 5 files changed, 160 insertions(+), 120 deletions(-) create mode 100644 src/port/pg_popcnt_choose.c create mode 100644 src/port/pg_popcnt_x86_64_accel.c diff --git a/src/port/Makefile b/src/port/Makefile index dcc8737e68..12c56b0ba7 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -44,6 +44,8 @@ OBJS = \ noblock.o \ path.o \ pg_bitutils.o \ + pg_popcnt_choose.o \ + pg_popcnt_x86_64_accel.o \ pg_strong_random.o \ pgcheckdir.o \ pgmkdirp.o \ diff --git a/src/port/meson.build b/src/port/meson.build index 92b593e6ef..ed8828c739 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -7,6 +7,8 @@ pgport_sources = [ 'noblock.c', 'path.c', 'pg_bitutils.c', + 'pg_popcnt_choose.c', + 'pg_popcnt_x86_64_accel.c', 'pg_strong_random.c', 'pgcheckdir.c', 'pgmkdirp.c', diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c index 640a89561a..d8b045d0a4 100644 --- a/src/port/pg_bitutils.c +++ b/src/port/pg_bitutils.c @@ -12,13 +12,6 @@ */ #include "c.h" -#ifdef HAVE__GET_CPUID -#include -#endif -#ifdef HAVE__CPUID -#include -#endif - #include "port/pg_bitutils.h" @@ -103,123 +96,14 @@ const uint8 pg_number_of_ones[256] = { 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 }; -static int pg_popcount32_slow(uint32 word); -static int pg_popcount64_slow(uint64 word); - -#ifdef TRY_POPCNT_FAST -static bool pg_popcount_available(void); -static int pg_popcount32_choose(uint32 word); -static int pg_popcount64_choose(uint64 word); -static int pg_popcount32_fast(uint32 word); -static int pg_popcount64_fast(uint64 word); - -int (*pg_popcount32) (uint32 word) = pg_popcount32_choose; -int (*pg_popcount64) (uint64 word) = pg_popcount64_choose; -#endif /* TRY_POPCNT_FAST */ - -#ifdef TRY_POPCNT_FAST - -/* - * Return true if CPUID indicates that the POPCNT instruction is available. - */ -static bool -pg_popcount_available(void) -{ - unsigned int exx[4] = {0, 0, 0, 0}; - -#if defined(HAVE__GET_CPUID) - __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); -#elif defined(HAVE__CPUID) - __cpuid(exx, 1); -#else -#error cpuid instruction not available -#endif - - return (exx[2] & (1 << 23)) != 0; /* POPCNT */ -} - -/* - * These functions get called on the first call to pg_popcount32 etc. - * They detect whether we can use the asm implementations, and replace - * the function pointers so that subsequent calls are routed directly to - * the chosen implementation. - */ -static int -pg_popcount32_choose(uint32 word) -{ - if (pg_popcount_available()) - { - pg_popcount32 = pg_popcount32_fast; - pg_popcount64 = pg_popcount64_fast; - } - else - { - pg_popcount32 = pg_popcount32_slow; - pg_popcount64 = pg_popcount64_slow; - } - - return pg_popcount32(word); -} - -static int -pg_popcount64_choose(uint64 word) -{ - if (pg_popcount_available()) - { - pg_popcount32 = pg_popcount32_fast; - pg_popcount64 = pg_popcount64_fast; - } - else - { - pg_popcount32 = pg_popcount32_slow; - pg_popcount64 = pg_popcount64_slow; - } - - return pg_popcount64(word); -} - -/* - * pg_popcount32_fast - * Return the number of 1 bits set in word - */ -static int -pg_popcount32_fast(uint32 word) -{ -#ifdef _MSC_VER - return __popcnt(word); -#else - uint32 res; - -__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc"); - return (int) res; -#endif -} - -/* - * pg_popcount64_fast - * Return the number of 1 bits set in word - */ -static int -pg_popcount64_fast(uint64 word) -{ -#ifdef _MSC_VER - return __popcnt64(word); -#else - uint64 res; - -__asm__ __volatile__(" popcntq %1,%0\n":"=q"(res):"rm"(word):"cc"); - return (int) res; -#endif -} - -#endif /* TRY_POPCNT_FAST */ - +int pg_popcount32_slow(uint32 word); +int pg_popcount64_slow(uint64 word); /* * pg_popcount32_slow * Return the number of 1 bits set in word */ -static int +int pg_popcount32_slow(uint32 word) { #ifdef HAVE__BUILTIN_POPCOUNT @@ -241,7 +125,7 @@ pg_popcount32_slow(uint32 word) * pg_popcount64_slow * Return the number of 1 bits set in word */ -static int +int pg_popcount64_slow(uint64 word) { #ifdef HAVE__BUILTIN_POPCOUNT diff --git a/src/port/pg_popcnt_choose.c b/src/port/pg_popcnt_choose.c new file mode 100644 index 0000000000..89fcf2609c --- /dev/null +++ b/src/port/pg_popcnt_choose.c @@ -0,0 +1,97 @@ +/*------------------------------------------------------------------------- + * + * pg_popcnt_choose.c + * For FAST operations, these methods do runtime checks and set the + * appropriate function pointers. + * + * Copyright (c) 2019-2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_popcnt_choose.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" + +#ifdef HAVE__GET_CPUID +#include +#endif +#ifdef HAVE__CPUID +#include +#endif + +#include "port/pg_bitutils.h" + + +/* In pg_bitutils.c file */ +int pg_popcount32_slow(uint32 word); +int pg_popcount64_slow(uint64 word); + +#ifdef TRY_POPCNT_FAST +static bool pg_popcount_available(void); +static int pg_popcount32_choose(uint32 word); +static int pg_popcount64_choose(uint64 word); + +/* In pg_popcnt_*_accel source file. */ +int pg_popcount32_fast(uint32 word); +int pg_popcount64_fast(uint64 word); + +int (*pg_popcount32) (uint32 word) = pg_popcount32_choose; +int (*pg_popcount64) (uint64 word) = pg_popcount64_choose; +#endif /* TRY_POPCNT_FAST */ + +#ifdef TRY_POPCNT_FAST + +/* + * Return true if CPUID indicates that the POPCNT instruction is available. + */ +static bool +pg_popcount_available(void) +{ + unsigned int exx[4] = {0, 0, 0, 0}; + +#if defined(HAVE__GET_CPUID) + __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); +#elif defined(HAVE__CPUID) + __cpuid(exx, 1); +#else +#error cpuid instruction not available +#endif + + return (exx[2] & (1 << 23)) != 0; /* POPCNT */ +} + +/* + * These functions get called on the first call to pg_popcount32 etc. + * They detect whether we can use the asm implementations, and replace + * the function pointers so that subsequent calls are routed directly to + * the chosen implementation. + */ +static void setup_function_pointers() +{ + if (pg_popcount_available()) + { + pg_popcount32 = pg_popcount32_fast; + pg_popcount64 = pg_popcount64_fast; + } + else + { + pg_popcount32 = pg_popcount32_slow; + pg_popcount64 = pg_popcount64_slow; + } +} + +static int +pg_popcount32_choose(uint32 word) +{ + setup_function_pointers(); + return pg_popcount32(word); +} + +static int +pg_popcount64_choose(uint64 word) +{ + setup_function_pointers(); + return pg_popcount64(word); +} +#endif /* TRY_POPCNT_FAST */ diff --git a/src/port/pg_popcnt_x86_64_accel.c b/src/port/pg_popcnt_x86_64_accel.c new file mode 100644 index 0000000000..2e9b2ee774 --- /dev/null +++ b/src/port/pg_popcnt_x86_64_accel.c @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------- + * + * pg_popcnt_x86_64_accel.c + * Fast POPCNT methods for x86_64. + * + * Copyright (c) 2019-2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_popcnt_x86_64_accel.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" + +#include "port/pg_bitutils.h" + +int pg_popcount32_fast(uint32 word); +int pg_popcount64_fast(uint64 word); + +#ifdef TRY_POPCNT_FAST +/* + * pg_popcount32_fast + * Return the number of 1 bits set in word + */ +int +pg_popcount32_fast(uint32 word) +{ +#ifdef _MSC_VER + return __popcnt(word); +#else + uint32 res; + +__asm__ __volatile__(" popcntl %1,%0\n":"=q"(res):"rm"(word):"cc"); + return (int) res; +#endif +} + +/* + * pg_popcount64_fast + * Return the number of 1 bits set in word + */ +int +pg_popcount64_fast(uint64 word) +{ +#ifdef _MSC_VER + return __popcnt64(word); +#else + uint64 res; + +__asm__ __volatile__(" popcntq %1,%0\n":"=q"(res):"rm"(word):"cc"); + return (int) res; +#endif +} + +#endif /* TRY_POPCNT_FAST */ -- 2.34.1