From 25cc8623d65d333e68cb43a792ba3055bf89b7c9 Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Fri, 2 Dec 2022 15:27:06 +0900 Subject: [PATCH v12 4/7] Use bitmapword for node-125 --- src/backend/lib/radixtree.c | 71 +++++++++++++++------------------- src/backend/nodes/bitmapset.c | 38 ------------------ src/include/nodes/bitmapset.h | 22 +---------- src/include/port/pg_bitutils.h | 58 +++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 98 deletions(-) diff --git a/src/backend/lib/radixtree.c b/src/backend/lib/radixtree.c index e7f61fd943..673cc5e46b 100644 --- a/src/backend/lib/radixtree.c +++ b/src/backend/lib/radixtree.c @@ -207,6 +207,9 @@ typedef struct rt_node_base125 /* The index of slots for each fanout */ uint8 slot_idxs[RT_NODE_MAX_SLOTS]; + + /* isset is a bitmap to track which slot is in use */ + bitmapword isset[WORDNUM(128)]; } rt_node_base_125; typedef struct rt_node_base256 @@ -271,9 +274,6 @@ typedef struct rt_node_leaf_125 { rt_node_base_125 base; - /* isset is a bitmap to track which slot is in use */ - uint8 isset[RT_NODE_NSLOTS_BITS(128)]; - /* number of values depends on size class */ uint64 values[FLEXIBLE_ARRAY_MEMBER]; } rt_node_leaf_125; @@ -655,13 +655,14 @@ node_125_is_chunk_used(rt_node_base_125 *node, uint8 chunk) return node->slot_idxs[chunk] != RT_NODE_125_INVALID_IDX; } +#ifdef USE_ASSERT_CHECKING /* Is the slot in the node used? */ static inline bool node_inner_125_is_slot_used(rt_node_inner_125 *node, uint8 slot) { Assert(!NODE_IS_LEAF(node)); Assert(slot < node->base.n.fanout); - return (node->children[slot] != NULL); + return (node->base.isset[WORDNUM(slot)] & ((bitmapword) 1 << BITNUM(slot))) != 0; } static inline bool @@ -669,8 +670,9 @@ node_leaf_125_is_slot_used(rt_node_leaf_125 *node, uint8 slot) { Assert(NODE_IS_LEAF(node)); Assert(slot < node->base.n.fanout); - return ((node->isset[RT_NODE_BITMAP_BYTE(slot)] & RT_NODE_BITMAP_BIT(slot)) != 0); + return (node->base.isset[WORDNUM(slot)] & ((bitmapword) 1 << BITNUM(slot))) != 0; } +#endif static inline rt_node * node_inner_125_get_child(rt_node_inner_125 *node, uint8 chunk) @@ -690,7 +692,10 @@ node_leaf_125_get_value(rt_node_leaf_125 *node, uint8 chunk) static void node_inner_125_delete(rt_node_inner_125 *node, uint8 chunk) { + int slotpos = node->base.slot_idxs[chunk]; + Assert(!NODE_IS_LEAF(node)); + node->base.isset[WORDNUM(slotpos)] &= ~((bitmapword) 1 << BITNUM(slotpos)); node->children[node->base.slot_idxs[chunk]] = NULL; node->base.slot_idxs[chunk] = RT_NODE_125_INVALID_IDX; } @@ -701,44 +706,35 @@ node_leaf_125_delete(rt_node_leaf_125 *node, uint8 chunk) int slotpos = node->base.slot_idxs[chunk]; Assert(NODE_IS_LEAF(node)); - node->isset[RT_NODE_BITMAP_BYTE(slotpos)] &= ~(RT_NODE_BITMAP_BIT(slotpos)); + node->base.isset[WORDNUM(slotpos)] &= ~((bitmapword) 1 << BITNUM(slotpos)); node->base.slot_idxs[chunk] = RT_NODE_125_INVALID_IDX; } /* Return an unused slot in node-125 */ static int -node_inner_125_find_unused_slot(rt_node_inner_125 *node, uint8 chunk) -{ - int slotpos = 0; - - Assert(!NODE_IS_LEAF(node)); - while (node_inner_125_is_slot_used(node, slotpos)) - slotpos++; - - return slotpos; -} - -static int -node_leaf_125_find_unused_slot(rt_node_leaf_125 *node, uint8 chunk) -{ - int slotpos; - - Assert(NODE_IS_LEAF(node)); +node_125_find_unused_slot(bitmapword *isset) + { + int slotpos; + int idx; + bitmapword inverse; - /* We iterate over the isset bitmap per byte then check each bit */ - for (slotpos = 0; slotpos < RT_NODE_NSLOTS_BITS(128); slotpos++) + /* get the first word with at least one bit not set */ + for (idx = 0; idx < WORDNUM(128); idx++) { - if (node->isset[slotpos] < 0xFF) - break; + if (isset[idx] < ~((bitmapword) 0)) + break; } - Assert(slotpos < RT_NODE_NSLOTS_BITS(128)); - slotpos *= BITS_PER_BYTE; - while (node_leaf_125_is_slot_used(node, slotpos)) - slotpos++; + /* To get the first unset bit in X, get the first set bit in ~X */ + inverse = ~(isset[idx]); + slotpos = idx * BITS_PER_BITMAPWORD; + slotpos += bmw_rightmost_one_pos(inverse); + + /* mark the slot used */ + isset[idx] |= RIGHTMOST_ONE(inverse); return slotpos; -} + } static inline void node_inner_125_insert(rt_node_inner_125 *node, uint8 chunk, rt_node *child) @@ -747,8 +743,7 @@ node_inner_125_insert(rt_node_inner_125 *node, uint8 chunk, rt_node *child) Assert(!NODE_IS_LEAF(node)); - /* find unused slot */ - slotpos = node_inner_125_find_unused_slot(node, chunk); + slotpos = node_125_find_unused_slot(node->base.isset); Assert(slotpos < node->base.n.fanout); node->base.slot_idxs[chunk] = slotpos; @@ -763,12 +758,10 @@ node_leaf_125_insert(rt_node_leaf_125 *node, uint8 chunk, uint64 value) Assert(NODE_IS_LEAF(node)); - /* find unused slot */ - slotpos = node_leaf_125_find_unused_slot(node, chunk); + slotpos = node_125_find_unused_slot(node->base.isset); Assert(slotpos < node->base.n.fanout); node->base.slot_idxs[chunk] = slotpos; - node->isset[RT_NODE_BITMAP_BYTE(slotpos)] |= RT_NODE_BITMAP_BIT(slotpos); node->values[slotpos] = value; } @@ -2395,9 +2388,9 @@ rt_dump_node(rt_node *node, int level, bool recurse) rt_node_leaf_125 *n = (rt_node_leaf_125 *) node; fprintf(stderr, ", isset-bitmap:"); - for (int i = 0; i < 16; i++) + for (int i = 0; i < WORDNUM(128); i++) { - fprintf(stderr, "%X ", (uint8) n->isset[i]); + fprintf(stderr, UINT64_FORMAT_HEX " ", n->base.isset[i]); } fprintf(stderr, "\n"); } diff --git a/src/backend/nodes/bitmapset.c b/src/backend/nodes/bitmapset.c index b7b274aeff..3fe0fd88ce 100644 --- a/src/backend/nodes/bitmapset.c +++ b/src/backend/nodes/bitmapset.c @@ -23,49 +23,11 @@ #include "common/hashfn.h" #include "nodes/bitmapset.h" #include "nodes/pg_list.h" -#include "port/pg_bitutils.h" -#define WORDNUM(x) ((x) / BITS_PER_BITMAPWORD) -#define BITNUM(x) ((x) % BITS_PER_BITMAPWORD) - #define BITMAPSET_SIZE(nwords) \ (offsetof(Bitmapset, words) + (nwords) * sizeof(bitmapword)) -/*---------- - * This is a well-known cute trick for isolating the rightmost one-bit - * in a word. It assumes two's complement arithmetic. Consider any - * nonzero value, and focus attention on the rightmost one. The value is - * then something like - * xxxxxx10000 - * where x's are unspecified bits. The two's complement negative is formed - * by inverting all the bits and adding one. Inversion gives - * yyyyyy01111 - * where each y is the inverse of the corresponding x. Incrementing gives - * yyyyyy10000 - * and then ANDing with the original value gives - * 00000010000 - * This works for all cases except original value = zero, where of course - * we get zero. - *---------- - */ -#define RIGHTMOST_ONE(x) ((signedbitmapword) (x) & -((signedbitmapword) (x))) - -#define HAS_MULTIPLE_ONES(x) ((bitmapword) RIGHTMOST_ONE(x) != (x)) - -/* Select appropriate bit-twiddling functions for bitmap word size */ -#if BITS_PER_BITMAPWORD == 32 -#define bmw_leftmost_one_pos(w) pg_leftmost_one_pos32(w) -#define bmw_rightmost_one_pos(w) pg_rightmost_one_pos32(w) -#define bmw_popcount(w) pg_popcount32(w) -#elif BITS_PER_BITMAPWORD == 64 -#define bmw_leftmost_one_pos(w) pg_leftmost_one_pos64(w) -#define bmw_rightmost_one_pos(w) pg_rightmost_one_pos64(w) -#define bmw_popcount(w) pg_popcount64(w) -#else -#error "invalid BITS_PER_BITMAPWORD" -#endif - /* * bms_copy - make a palloc'd copy of a bitmapset diff --git a/src/include/nodes/bitmapset.h b/src/include/nodes/bitmapset.h index 2792281658..06fa21ccaa 100644 --- a/src/include/nodes/bitmapset.h +++ b/src/include/nodes/bitmapset.h @@ -21,33 +21,13 @@ #define BITMAPSET_H #include "nodes/nodes.h" +#include "port/pg_bitutils.h" /* * Forward decl to save including pg_list.h */ struct List; -/* - * Data representation - * - * Larger bitmap word sizes generally give better performance, so long as - * they're not wider than the processor can handle efficiently. We use - * 64-bit words if pointers are that large, else 32-bit words. - */ -#if SIZEOF_VOID_P >= 8 - -#define BITS_PER_BITMAPWORD 64 -typedef uint64 bitmapword; /* must be an unsigned type */ -typedef int64 signedbitmapword; /* must be the matching signed type */ - -#else - -#define BITS_PER_BITMAPWORD 32 -typedef uint32 bitmapword; /* must be an unsigned type */ -typedef int32 signedbitmapword; /* must be the matching signed type */ - -#endif - typedef struct Bitmapset { pg_node_attr(custom_copy_equal, special_read_write) diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h index 814e0b2dba..ad5aa2c5cf 100644 --- a/src/include/port/pg_bitutils.h +++ b/src/include/port/pg_bitutils.h @@ -17,6 +17,51 @@ extern PGDLLIMPORT const uint8 pg_leftmost_one_pos[256]; extern PGDLLIMPORT const uint8 pg_rightmost_one_pos[256]; extern PGDLLIMPORT const uint8 pg_number_of_ones[256]; +/* + * Platform-specific types + * + * Larger bitmap word sizes generally give better performance, so long as + * they're not wider than the processor can handle efficiently. We use + * 64-bit words if pointers are that large, else 32-bit words. + */ +#if SIZEOF_VOID_P >= 8 + +#define BITS_PER_BITMAPWORD 64 +typedef uint64 bitmapword; /* must be an unsigned type */ +typedef int64 signedbitmapword; /* must be the matching signed type */ + +#else + +#define BITS_PER_BITMAPWORD 32 +typedef uint32 bitmapword; /* must be an unsigned type */ +typedef int32 signedbitmapword; /* must be the matching signed type */ + +#endif + +#define WORDNUM(x) ((x) / BITS_PER_BITMAPWORD) +#define BITNUM(x) ((x) % BITS_PER_BITMAPWORD) + +/*---------- + * This is a well-known cute trick for isolating the rightmost one-bit + * in a word. It assumes two's complement arithmetic. Consider any + * nonzero value, and focus attention on the rightmost one. The value is + * then something like + * xxxxxx10000 + * where x's are unspecified bits. The two's complement negative is formed + * by inverting all the bits and adding one. Inversion gives + * yyyyyy01111 + * where each y is the inverse of the corresponding x. Incrementing gives + * yyyyyy10000 + * and then ANDing with the original value gives + * 00000010000 + * This works for all cases except original value = zero, where of course + * we get zero. + *---------- + */ +#define RIGHTMOST_ONE(x) ((signedbitmapword) (x) & -((signedbitmapword) (x))) + +#define HAS_MULTIPLE_ONES(x) ((bitmapword) RIGHTMOST_ONE(x) != (x)) + /* * pg_leftmost_one_pos32 * Returns the position of the most significant set bit in "word", @@ -291,4 +336,17 @@ pg_rotate_left32(uint32 word, int n) #define pg_prevpower2_size_t pg_prevpower2_64 #endif +/* variants of some functions for bitmap word size */ +#if BITS_PER_BITMAPWORD == 32 +#define bmw_leftmost_one_pos pg_leftmost_one_pos32 +#define bmw_rightmost_one_pos pg_rightmost_one_pos32 +#define bmw_popcount pg_popcount32 +#elif BITS_PER_BITMAPWORD == 64 +#define bmw_leftmost_one_pos pg_leftmost_one_pos64 +#define bmw_rightmost_one_pos pg_rightmost_one_pos64 +#define bmw_popcount pg_popcount64 +#else +#error "invalid BITS_PER_BITMAPWORD" +#endif + #endif /* PG_BITUTILS_H */ -- 2.31.1