From ebd62356210eff2f38772a9c46a0a8792c0e9ce3 Mon Sep 17 00:00:00 2001 From: Maxime Schoemans Date: Mon, 20 Mar 2023 11:48:05 -0400 Subject: [PATCH v2] Apply division before adding remainder --- src/backend/utils/adt/multirangetypes_selfuncs.c | 5 ++++- src/backend/utils/adt/rangetypes_selfuncs.c | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/multirangetypes_selfuncs.c b/src/backend/utils/adt/multirangetypes_selfuncs.c index 7ba4aa8b04..ad14b789f4 100644 --- a/src/backend/utils/adt/multirangetypes_selfuncs.c +++ b/src/backend/utils/adt/multirangetypes_selfuncs.c @@ -1412,11 +1412,14 @@ calc_hist_join_selectivity(TypeCacheEntry *typcache, prev_sel2 = cur_sel2; } + /* P(X < Y) = 0.5 * Sum(...) */ + selectivity /= 2; + /* Include remainder of hist2 if any */ if (j < nhist2) selectivity += 1 - prev_sel2; - return selectivity / 2; + return selectivity; } /* diff --git a/src/backend/utils/adt/rangetypes_selfuncs.c b/src/backend/utils/adt/rangetypes_selfuncs.c index 007e14bcf6..90970943b3 100644 --- a/src/backend/utils/adt/rangetypes_selfuncs.c +++ b/src/backend/utils/adt/rangetypes_selfuncs.c @@ -1342,11 +1342,14 @@ calc_hist_join_selectivity(TypeCacheEntry *typcache, prev_sel2 = cur_sel2; } + /* P(X < Y) = 0.5 * Sum(...) */ + selectivity /= 2; + /* Include remainder of hist2 if any */ if (j < nhist2) selectivity += 1 - prev_sel2; - return selectivity / 2; + return selectivity; } /* -- 2.17.1