From 27c6f4b6e947af27ccc9f6ae881407010f141bbe Mon Sep 17 00:00:00 2001 From: Ilia Evdokimov Date: Mon, 10 Nov 2025 17:06:34 +0300 Subject: [PATCH v4 2/2] Optimize MCV matching in eqjoinsel_inner() and eqjoinsel_semi() Previously, MCV values from both sides of a join were compared using an O(N^2) nested-loop algorithm. When default_statistics_target was set to large values, this could make query planning noticeably slower. This patch introduces a hash-based O(N) algorithm for matching MCVs. The planner now switches to the hash method when the MCV lists are large enough to amortize hash setup costs, while keeping the old nested-loop path for small lists. The threshold is currently set to 100 entries. For data types that do not support hashing, the code still falls back to the O(N^2) algorithm. Author: David Geier Author: Ilia Evdokimov Reviewed-by: Tom Lane --- src/backend/utils/adt/selfuncs.c | 211 +++++++++++++++++++++++++++++-- 1 file changed, 202 insertions(+), 9 deletions(-) diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 55cd0486bf9..b22e2e5d920 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -143,12 +143,20 @@ #define DEFAULT_PAGE_CPU_MULTIPLIER 50.0 +/* + * Switch to hash-based MCV matching when lists are large enough + * to amortize hash setup cost. + */ +#define EQJOINSEL_MCV_HASH_THRESHOLD 100 + +struct McvHashTable_hash; + /* Hooks for plugins to get control when we ask for stats */ get_relation_stats_hook_type get_relation_stats_hook = NULL; get_index_stats_hook_type get_index_stats_hook = NULL; static double eqsel_internal(PG_FUNCTION_ARGS, bool negate); -static double eqjoinsel_inner(Oid opfuncoid, Oid collation, +static double eqjoinsel_inner(Oid operator, Oid collation, VariableStatData *vardata1, VariableStatData *vardata2, double nd1, double nd2, bool isdefault1, bool isdefault2, @@ -157,7 +165,7 @@ static double eqjoinsel_inner(Oid opfuncoid, Oid collation, bool have_mcvs1, bool have_mcvs2, double *matchfreq_mcvs1, double *matchfreq_mcvs2, int *nmatches_mcvs); -static double eqjoinsel_semi(Oid opfuncoid, Oid collation, +static double eqjoinsel_semi(Oid operator, Oid opfuncoid, Oid collation, VariableStatData *vardata1, VariableStatData *vardata2, double nd1, double nd2, bool isdefault1, bool isdefault2, @@ -220,6 +228,55 @@ static bool get_actual_variable_endpoint(Relation heapRel, static RelOptInfo *find_join_input_rel(PlannerInfo *root, Relids relids); static double btcost_correlation(IndexOptInfo *index, VariableStatData *vardata); +static uint32 hash_mcv(struct McvHashTable_hash *hashTable, Datum key); +static bool are_mcvs_equal(struct McvHashTable_hash *hashTable, Datum value1, Datum value2); + +typedef struct McvHashEntry +{ + Datum value; + uint32 index; + uint32 hash; + char status; +} McvHashEntry; + +typedef struct McvHashContext +{ + FmgrInfo equal_proc; + FmgrInfo hash_proc; + Oid collation; +} McvHashContext; + +#define SH_PREFIX McvHashTable +#define SH_ELEMENT_TYPE McvHashEntry +#define SH_KEY_TYPE Datum +#define SH_KEY value +#define SH_HASH_KEY(mcvs, key) hash_mcv(mcvs, key) +#define SH_EQUAL(mcvs, key0, key1) are_mcvs_equal(mcvs, key0, key1) +#define SH_SCOPE static inline +#define SH_STORE_HASH +#define SH_GET_HASH(mcvs, key) key->hash +#define SH_DEFINE +#define SH_DECLARE +#include "lib/simplehash.h" + +static uint32 +hash_mcv(struct McvHashTable_hash *hashTable, Datum key) +{ + McvHashContext *context = (McvHashContext *)hashTable->private_data; + return DatumGetUInt32(FunctionCall1Coll(&context->hash_proc, context->collation, key)); +} + +static bool +are_mcvs_equal(struct McvHashTable_hash *hashTable, Datum value1, Datum value2) +{ + /* + * We can safely use FunctionCall2Coll() which requires the result to + * never be NULL, because MCV arrays from 'pg_statistic' don't contain + * NULL values + */ + McvHashContext *context = (McvHashContext *)hashTable->private_data; + return DatumGetBool(FunctionCall2Coll(&context->equal_proc, context->collation, value1, value2)); +} /* @@ -2367,7 +2424,7 @@ eqjoinsel(PG_FUNCTION_ARGS) } /* We need to compute the inner-join selectivity in all cases */ - selec_inner = eqjoinsel_inner(opfuncoid, collation, + selec_inner = eqjoinsel_inner(operator, collation, &vardata1, &vardata2, nd1, nd2, isdefault1, isdefault2, @@ -2396,7 +2453,7 @@ eqjoinsel(PG_FUNCTION_ARGS) inner_rel = find_join_input_rel(root, sjinfo->min_righthand); if (!join_is_reversed) - selec = eqjoinsel_semi(opfuncoid, collation, + selec = eqjoinsel_semi(operator, opfuncoid, collation, &vardata1, &vardata2, nd1, nd2, isdefault1, isdefault2, @@ -2410,7 +2467,7 @@ eqjoinsel(PG_FUNCTION_ARGS) Oid commop = get_commutator(operator); Oid commopfuncoid = OidIsValid(commop) ? get_opcode(commop) : InvalidOid; - selec = eqjoinsel_semi(commopfuncoid, collation, + selec = eqjoinsel_semi(operator, commopfuncoid, collation, &vardata2, &vardata1, nd2, nd1, isdefault2, isdefault1, @@ -2459,7 +2516,7 @@ eqjoinsel(PG_FUNCTION_ARGS) * that it's worth trying to distinguish them here. */ static double -eqjoinsel_inner(Oid opfuncoid, Oid collation, +eqjoinsel_inner(Oid operator, Oid collation, VariableStatData *vardata1, VariableStatData *vardata2, double nd1, double nd2, bool isdefault1, bool isdefault2, @@ -2502,8 +2559,11 @@ eqjoinsel_inner(Oid opfuncoid, Oid collation, totalsel2; int i, nmatches; + Oid hashLeft = InvalidOid; + Oid hashRight = InvalidOid; - fmgr_info(opfuncoid, &eqproc); + fmgr_info(get_opcode(operator), &eqproc); + get_op_hash_functions(operator, &hashLeft, &hashRight); /* * Save a few cycles by setting up the fcinfo struct just once. Using @@ -2527,6 +2587,70 @@ eqjoinsel_inner(Oid opfuncoid, Oid collation, */ matchprodfreq = 0.0; nmatches = 0; + + /* + * If one MCV array contains less than 100 values, there's no gain in using a hash table. + * The sweet spot of using hash table lookups instead of iterating is slightly higher + * than 1 but we don't bother here because the gains are neglectable. + */ + if (OidIsValid(hashLeft) && hashLeft == hashRight && + Min(sslot1->nvalues, sslot2->nvalues) > EQJOINSEL_MCV_HASH_THRESHOLD) + { + AttStatsSlot *statsInner = sslot2; + AttStatsSlot *statsOuter = sslot1; + bool *hasMatchInner = hasmatch2; + bool *hasMatchOuter = hasmatch1; + int nvaluesInner = sslot2->nvalues; + int nvaluesOuter = sslot1->nvalues; + McvHashContext hashContext; + McvHashTable_hash *hashTable; + + /* Make sure we build the hash table on the smaller array. */ + if (sslot1->nvalues < sslot2->nvalues) + { + statsInner = sslot1; + statsOuter = sslot2; + hasMatchInner = hasmatch1; + hasMatchOuter = hasmatch2; + nvaluesInner = sslot1->nvalues; + nvaluesOuter = sslot2->nvalues; + } + + /* 1. Create hash table of smaller 'pg_statistic' array. That's O(n). */ + fmgr_info(get_opcode(operator), &hashContext.equal_proc); + fmgr_info(hashLeft, &hashContext.hash_proc); /* hashLeft == hashRight */ + hashContext.collation = collation; + + hashTable = McvHashTable_create(CurrentMemoryContext, nvaluesInner, &hashContext); + + for (i = 0; i < nvaluesInner; i++) + { + bool found = false; + McvHashEntry *entry = McvHashTable_insert(hashTable, statsInner->values[i], &found); + + Assert(!found); + + entry->index = i; + } + + /* 2. Look-up values from other 'pg_statistic' array against hash map to find matches. */ + for (i = 0; i < nvaluesOuter; i++) + { + McvHashEntry *entry = McvHashTable_lookup(hashTable, statsOuter->values[i]); + + if (entry != NULL) + { + hasMatchInner[entry->index] = hasMatchOuter[i] = true; + nmatches++; + matchprodfreq += statsInner->numbers[entry->index] * statsOuter->numbers[i]; + } + } + + McvHashTable_destroy(hashTable); + } + else + { + /* Fallback to O(N^2) algorithm if hash based variant didn't succeed. */ for (i = 0; i < sslot1->nvalues; i++) { int j; @@ -2551,6 +2675,7 @@ eqjoinsel_inner(Oid opfuncoid, Oid collation, } } } + } CLAMP_PROBABILITY(matchprodfreq); /* Sum up frequencies of matched and unmatched MCVs */ matchfreq1 = unmatchfreq1 = 0.0; @@ -2663,7 +2788,7 @@ eqjoinsel_inner(Oid opfuncoid, Oid collation, * Unlike eqjoinsel_inner, we have to cope with opfuncoid being InvalidOid. */ static double -eqjoinsel_semi(Oid opfuncoid, Oid collation, +eqjoinsel_semi(Oid operator, Oid opfuncoid, Oid collation, VariableStatData *vardata1, VariableStatData *vardata2, double nd1, double nd2, bool isdefault1, bool isdefault2, @@ -2744,7 +2869,11 @@ eqjoinsel_semi(Oid opfuncoid, Oid collation, */ if (clamped_nvalues2 != sslot2->nvalues) { - fmgr_info(opfuncoid, &eqproc); + Oid hashLeft = InvalidOid; + Oid hashRight = InvalidOid; + + fmgr_info(get_opcode(operator), &eqproc); + get_op_hash_functions(operator, &hashLeft, &hashRight); /* * Save a few cycles by setting up the fcinfo struct just once. Using @@ -2767,6 +2896,69 @@ eqjoinsel_semi(Oid opfuncoid, Oid collation, * and because the math wouldn't add up... */ nmatches = 0; + + /* + * If one MCV array contains less than 100 values, there's no gain in using a hash table. + * The sweet spot of using hash table lookups instead of iterating is slightly higher + * than 1 but we don't bother here because the gains are neglectable. + */ + if (OidIsValid(hashLeft) && hashLeft == hashRight && + Min(sslot1->nvalues, clamped_nvalues2) > EQJOINSEL_MCV_HASH_THRESHOLD) + { + AttStatsSlot *statsInner = sslot2; + AttStatsSlot *statsOuter = sslot1; + bool *hasMatchInner = hasmatch2; + bool *hasMatchOuter = hasmatch1; + int nvaluesInner = clamped_nvalues2; + int nvaluesOuter = sslot1->nvalues; + McvHashContext hashContext; + McvHashTable_hash *hashTable; + + /* Make sure we build the hash table on the smaller array. */ + if (sslot1->nvalues < clamped_nvalues2) + { + statsInner = sslot1; + statsOuter = sslot2; + hasMatchInner = hasmatch1; + hasMatchOuter = hasmatch2; + nvaluesInner = sslot1->nvalues; + nvaluesOuter = clamped_nvalues2; + } + + /* 1. Create hash table of smaller 'pg_statistic' array. That's O(n). */ + fmgr_info(get_opcode(operator), &hashContext.equal_proc); + fmgr_info(hashLeft, &hashContext.hash_proc); /* hashLeft == hashRight */ + hashContext.collation = collation; + + hashTable = McvHashTable_create(CurrentMemoryContext, nvaluesInner, &hashContext); + + for (i = 0; i < nvaluesInner; i++) + { + bool found = false; + McvHashEntry *entry = McvHashTable_insert(hashTable, statsInner->values[i], &found); + + Assert(!found); + + entry->index = i; + } + + /* 2. Look-up values from other 'pg_statistic' array against hash map to find matches. */ + for (i = 0; i < nvaluesOuter; i++) + { + McvHashEntry *entry = McvHashTable_lookup(hashTable, statsOuter->values[i]); + + if (entry != NULL) + { + hasMatchInner[entry->index] = hasMatchOuter[i] = true; + nmatches++; + } + } + + McvHashTable_destroy(hashTable); + } + else + { + /* Fallback to O(N^2) algorithm if hash based variant didn't succeed. */ for (i = 0; i < sslot1->nvalues; i++) { int j; @@ -2790,6 +2982,7 @@ eqjoinsel_semi(Oid opfuncoid, Oid collation, } } } + } /* Sum up frequencies of matched MCVs */ matchfreq1 = 0.0; for (i = 0; i < sslot1->nvalues; i++) -- 2.34.1