>From 6e3e16f46f93f045c137c070b48e387a470c3a08 Mon Sep 17 00:00:00 2001
From: Tomas Vondra <tomas@pgaddict.com>
Date: Mon, 6 Apr 2015 19:42:18 +0200
Subject: [PATCH 3/9] clause reduction using functional dependencies

During planning, use functional dependencies to decide which clauses to
skip during cardinality estimation. Initial and rather simplistic
implementation.

This only works with regular WHERE clauses, not clauses used for join
clauses.

Note: The clause_is_mv_compatible() needs to identify the relation (so
that we can fetch the list of multivariate stats by OID).
planner_rt_fetch() seems like the appropriate way to get the relation
OID, but apparently it only works with simple vars. Maybe
examine_variable() would make this work with more complex vars too?

Includes regression tests analyzing functional dependencies (part of
ANALYZE) on several datasets (no dependencies, no transitive
dependencies, ...).

Checks that a query with conditions on two columns, where one (B) is
functionally dependent on the other one (A), correctly ignores the
clause on (B) and chooses bitmap index scan instead of plain index scan
(which is what happens otherwise, thanks to assumption of
independence).

Note: Functional dependencies only work with equality clauses, no
inequalities etc.
---
 src/backend/optimizer/path/clausesel.c        | 505 +++++++++++++++++++++++++-
 src/backend/utils/mvstats/README.dependencies |  63 ++--
 src/backend/utils/mvstats/README.stats        |  36 ++
 src/backend/utils/mvstats/common.c            |   5 +-
 src/backend/utils/mvstats/dependencies.c      |  24 ++
 src/include/utils/mvstats.h                   |   3 +-
 src/include/utils/rel.h                       |   2 +-
 src/test/regress/expected/mv_dependencies.out |  24 ++
 src/test/regress/parallel_schedule            |   3 +
 src/test/regress/sql/mv_dependencies.sql      |  15 +
 10 files changed, 637 insertions(+), 43 deletions(-)
 create mode 100644 src/backend/utils/mvstats/README.stats

diff --git a/src/backend/optimizer/path/clausesel.c b/src/backend/optimizer/path/clausesel.c
index 02660c2..a3afdf5 100644
--- a/src/backend/optimizer/path/clausesel.c
+++ b/src/backend/optimizer/path/clausesel.c
@@ -14,14 +14,19 @@
  */
 #include "postgres.h"
 
+#include "access/sysattr.h"
+#include "catalog/pg_operator.h"
 #include "nodes/makefuncs.h"
 #include "optimizer/clauses.h"
 #include "optimizer/cost.h"
 #include "optimizer/pathnode.h"
 #include "optimizer/plancat.h"
+#include "optimizer/var.h"
 #include "utils/fmgroids.h"
 #include "utils/lsyscache.h"
+#include "utils/mvstats.h"
 #include "utils/selfuncs.h"
+#include "utils/typcache.h"
 
 
 /*
@@ -41,6 +46,25 @@ typedef struct RangeQueryClause
 static void addRangeClause(RangeQueryClause **rqlist, Node *clause,
 			   bool varonleft, bool isLTsel, Selectivity s2);
 
+#define		MV_CLAUSE_TYPE_FDEP		0x01
+
+static bool clause_is_mv_compatible(Node *clause, Index relid, AttrNumber *attnum);
+
+static Bitmapset  *collect_mv_attnums(List *clauses, Index relid);
+
+static int count_mv_attnums(List *clauses, Index relid);
+
+static int count_varnos(List *clauses, Index *relid);
+
+static List *clauselist_apply_dependencies(PlannerInfo *root, List *clauses,
+									Index relid, List *stats);
+
+static bool has_stats(List *stats, int type);
+
+static List * find_stats(PlannerInfo *root, Index relid);
+
+static bool stats_type_matches(MVStatisticInfo *stat, int type);
+
 
 /****************************************************************************
  *		ROUTINES TO COMPUTE SELECTIVITIES
@@ -60,7 +84,19 @@ static void addRangeClause(RangeQueryClause **rqlist, Node *clause,
  * subclauses.  However, that's only right if the subclauses have independent
  * probabilities, and in reality they are often NOT independent.  So,
  * we want to be smarter where we can.
-
+ *
+ * The first thing we try to do is applying multivariate statistics, in a way
+ * that intends to minimize the overhead when there are no multivariate stats
+ * on the relation. Thus we do several simple (and inexpensive) checks first,
+ * to verify that suitable multivariate statistics exist.
+ *
+ * If we identify such multivariate statistics apply, we try to apply them.
+ * Currently we only have (soft) functional dependencies, so we try to reduce
+ * the list of clauses.
+ *
+ * Then we remove the clauses estimated using multivariate stats, and process
+ * the rest of the clauses using the regular per-column stats.
+ *
  * Currently, the only extra smarts we have is to recognize "range queries",
  * such as "x > 34 AND x < 42".  Clauses are recognized as possible range
  * query components if they are restriction opclauses whose operators have
@@ -99,6 +135,22 @@ clauselist_selectivity(PlannerInfo *root,
 	RangeQueryClause *rqlist = NULL;
 	ListCell   *l;
 
+	/* processing mv stats */
+	Oid			relid = InvalidOid;
+
+	/* list of multivariate stats on the relation */
+	List	   *stats = NIL;
+
+	/*
+	 * To fetch the statistics, we first need to determine the rel. Currently
+	 * point we only support estimates of simple restrictions with all Vars
+	 * referencing a single baserel. However set_baserel_size_estimates() sets
+	 * varRelid=0 so we have to actually inspect the clauses by pull_varnos
+	 * and see if there's just a single varno referenced.
+	 */
+	if ((count_varnos(clauses, &relid) == 1) && ((varRelid == 0) || (varRelid == relid)))
+		stats = find_stats(root, relid);
+
 	/*
 	 * If there's exactly one clause, then no use in trying to match up pairs,
 	 * so just go directly to clause_selectivity().
@@ -108,6 +160,24 @@ clauselist_selectivity(PlannerInfo *root,
 								  varRelid, jointype, sjinfo);
 
 	/*
+	 * Apply functional dependencies, but first check that there are some stats
+	 * with functional dependencies built (by simply walking the stats list),
+	 * and that there are at two or more attributes referenced by clauses that
+	 * may be reduced using functional dependencies.
+	 *
+	 * We would find that anyway when trying to actually apply the functional
+	 * dependencies, but let's do the cheap checks first.
+	 *
+	 * After applying the functional dependencies we get the remainig clauses
+	 * that need to be estimated by other types of stats (MCV, histograms etc).
+	 */
+	if (has_stats(stats, MV_CLAUSE_TYPE_FDEP) &&
+		(count_mv_attnums(clauses, relid) >= 2))
+	{
+		clauses = clauselist_apply_dependencies(root, clauses, relid, stats);
+	}
+
+	/*
 	 * Initial scan over clauses.  Anything that doesn't look like a potential
 	 * rangequery clause gets multiplied into s1 and forgotten. Anything that
 	 * does gets inserted into an rqlist entry.
@@ -763,3 +833,436 @@ clause_selectivity(PlannerInfo *root,
 
 	return s1;
 }
+
+/*
+ * Collect attributes from mv-compatible clauses.
+ */
+static Bitmapset *
+collect_mv_attnums(List *clauses, Index relid)
+{
+	Bitmapset  *attnums = NULL;
+	ListCell   *l;
+
+	/*
+	 * Walk through the clauses and identify the ones we can estimate
+	 * using multivariate stats, and remember the relid/columns. We'll
+	 * then cross-check if we have suitable stats, and only if needed
+	 * we'll split the clauses into multivariate and regular lists.
+	 *
+	 * For now we're only interested in RestrictInfo nodes with nested
+	 * OpExpr, using either a range or equality.
+	 */
+	foreach (l, clauses)
+	{
+		AttrNumber attnum;
+		Node	   *clause = (Node *) lfirst(l);
+
+		/* ignore the result for now - we only need the info */
+		if (clause_is_mv_compatible(clause, relid, &attnum))
+			attnums = bms_add_member(attnums, attnum);
+	}
+
+	/*
+	 * If there are not at least two attributes referenced by the clause(s),
+	 * we can throw everything out (as we'll revert to simple stats).
+	 */
+	if (bms_num_members(attnums) <= 1)
+	{
+		if (attnums != NULL)
+			pfree(attnums);
+		attnums = NULL;
+	}
+
+	return attnums;
+}
+
+/*
+ * Count the number of attributes in clauses compatible with multivariate stats.
+ */
+static int
+count_mv_attnums(List *clauses, Index relid)
+{
+	int c;
+	Bitmapset *attnums = collect_mv_attnums(clauses, relid);
+
+	c = bms_num_members(attnums);
+
+	bms_free(attnums);
+
+	return c;
+}
+
+/*
+ * Count varnos referenced in the clauses, and if there's a single varno then
+ * return the index in 'relid'.
+ */
+static int
+count_varnos(List *clauses, Index *relid)
+{
+	int cnt;
+	Bitmapset *varnos = NULL;
+
+	varnos = pull_varnos((Node *) clauses);
+	cnt = bms_num_members(varnos);
+
+	/* if there's a single varno in the clauses, remember it */
+	if (bms_num_members(varnos) == 1)
+		*relid = bms_singleton_member(varnos);
+
+	bms_free(varnos);
+
+	return cnt;
+}
+
+typedef struct
+{
+	Index		varno;		/* relid we're interested in */
+	Bitmapset  *varattnos;	/* attnums referenced by the clauses */
+} mv_compatible_context;
+
+/*
+ * Recursive walker that checks compatibility of the clause with multivariate
+ * statistics, and collects attnums from the Vars.
+ *
+ * XXX The original idea was to combine this with expression_tree_walker, but
+ *     I've been unable to make that work - seems that does not quite allow
+ *     checking the structure. Hence the explicit calls to the walker.
+ */
+static bool
+mv_compatible_walker(Node *node, mv_compatible_context *context)
+{
+	if (node == NULL)
+		return false;
+
+	if (IsA(node, RestrictInfo))
+	{
+		RestrictInfo *rinfo = (RestrictInfo *) node;
+
+		/* Pseudoconstants are not really interesting here. */
+		if (rinfo->pseudoconstant)
+			return true;
+
+		/* clauses referencing multiple varnos are incompatible */
+		if (bms_membership(rinfo->clause_relids) != BMS_SINGLETON)
+			return true;
+
+		/* check the clause inside the RestrictInfo */
+		return mv_compatible_walker((Node*)rinfo->clause, (void *) context);
+	}
+
+	if (IsA(node, Var))
+	{
+		Var * var = (Var*)node;
+
+		/*
+		 * Also, the variable needs to reference the right relid (this might be
+		 * unnecessary given the other checks, but let's be sure).
+		 */
+		if (var->varno != context->varno)
+			return true;
+
+		/* Also skip system attributes (we don't allow stats on those). */
+		if (! AttrNumberIsForUserDefinedAttr(var->varattno))
+			return true;
+
+		/* Seems fine, so let's remember the attnum. */
+		context->varattnos = bms_add_member(context->varattnos, var->varattno);
+
+		return false;
+	}
+
+	/*
+	 * And finally the operator expressions - we only allow simple expressions
+	 * with two arguments, where one is a Var and the other is a constant, and
+	 * it's a simple comparison (which we detect using estimator function).
+	 */
+	if (is_opclause(node))
+	{
+		OpExpr	   *expr = (OpExpr *) node;
+		Var		   *var;
+		bool		varonleft = true;
+		bool		ok;
+
+		/*
+		 * Only expressions with two arguments are considered compatible.
+		 *
+		 * XXX Possibly unnecessary (can OpExpr have different arg count?).
+		 */
+		if (list_length(expr->args) != 2)
+			return true;
+
+		/* see if it actually has the right */
+		ok = (NumRelids((Node*)expr) == 1) &&
+				(is_pseudo_constant_clause(lsecond(expr->args)) ||
+				(varonleft = false,
+				 is_pseudo_constant_clause(linitial(expr->args))));
+
+		/* unsupported structure (two variables or so) */
+		if (! ok)
+			return true;
+
+		/*
+		 * If it's not a "<" or ">" or "=" operator, just ignore the clause.
+		 * Otherwise note the relid and attnum for the variable. This uses the
+		 * function for estimating selectivity, ont the operator directly (a bit
+		 * awkward, but well ...).
+		 */
+		switch (get_oprrest(expr->opno))
+		{
+			case F_EQSEL:
+
+				/* equality conditions are compatible with all statistics */
+				break;
+
+			default:
+
+				/* unknown estimator */
+				return true;
+		}
+
+		var = (varonleft) ? linitial(expr->args) : lsecond(expr->args);
+
+		return mv_compatible_walker((Node *) var, context);
+	}
+
+	/* Node not explicitly supported, so terminate */
+	return true;
+}
+
+/*
+ * Determines whether the clause is compatible with multivariate stats,
+ * and if it is, returns some additional information - varno (index
+ * into simple_rte_array) and a bitmap of attributes. This is then
+ * used to fetch related multivariate statistics.
+ *
+ * At this moment we only support basic conditions of the form
+ *
+ *     variable OP constant
+ *
+ * where OP is one of [=,<,<=,>=,>] (which is however determined by
+ * looking at the associated function for estimating selectivity, just
+ * like with the single-dimensional case).
+ *
+ * TODO Support 'OR clauses' - shouldn't be all that difficult to
+ *      evaluate them using multivariate stats.
+ */
+static bool
+clause_is_mv_compatible(Node *clause, Index relid, AttrNumber *attnum)
+{
+	mv_compatible_context context;
+
+	context.varno = relid;
+	context.varattnos = NULL;	/* no attnums */
+
+	if (mv_compatible_walker(clause, (void *) &context))
+		return false;
+
+	/* remember the newly collected attnums */
+	*attnum = bms_singleton_member(context.varattnos);
+
+	return true;
+}
+
+
+/*
+ * Reduce clauses using functional dependencies
+ */
+static List*
+fdeps_reduce_clauses(List *clauses, Index relid, Bitmapset *reduced_attnums)
+{
+	ListCell *lc;
+	List	 *reduced_clauses = NIL;
+
+	foreach (lc, clauses)
+	{
+		AttrNumber attnum = InvalidAttrNumber;
+		Node * clause = (Node*)lfirst(lc);
+
+		/* ignore clauses that are not compatible with functional dependencies */
+		if (! clause_is_mv_compatible(clause, relid, &attnum))
+			reduced_clauses = lappend(reduced_clauses, clause);
+
+		/* for equality clauses, only keep those not on reduced attributes */
+		if (! bms_is_member(attnum, reduced_attnums))
+			reduced_clauses = lappend(reduced_clauses, clause);
+	}
+
+	return reduced_clauses;
+}
+
+/*
+ * decide which attributes are redundant (for equality clauses)
+ *
+ * We try to apply all functional dependencies available, and for each one we
+ * check if it matches attnums from equality clauses, but only those not yet
+ * reduced.
+ *
+ * XXX Not sure if the order in which we apply the dependencies matters.
+ *
+ * XXX We do not combine functional dependencies from separate stats. That is
+ *     if we have dependencies on [a,b] and [b,c], then we don't deduce
+ *     a->c from a->b and b->c. Computing such transitive closure is a possible
+ *     future improvement.
+ */
+static Bitmapset *
+fdeps_reduce_attnums(List *stats, Bitmapset *attnums)
+{
+	ListCell  *lc;
+	Bitmapset *reduced = NULL;
+
+	foreach (lc, stats)
+	{
+		int i;
+		MVDependencies dependencies = NULL;
+		MVStatisticInfo *info = (MVStatisticInfo *)lfirst(lc);
+
+		/* skip statistics without dependencies */
+		if (! stats_type_matches(info, MV_CLAUSE_TYPE_FDEP))
+			continue;
+
+		/* fetch and deserialize dependencies */
+		dependencies = load_mv_dependencies(info->mvoid);
+
+		for (i = 0; i < dependencies->ndeps; i++)
+		{
+			int j;
+			bool matched = true;
+			MVDependency dep = dependencies->deps[i];
+
+			/* we don't bother to break the loop early (only few attributes) */
+			for (j = 0; j < dep->nattributes; j++)
+			{
+				if (! bms_is_member(dep->attributes[j], attnums))
+					matched = false;
+
+				if (bms_is_member(dep->attributes[j], reduced))
+					matched = false;
+			}
+
+			/* if dependency applies, mark the last attribute as reduced */
+			if (matched)
+				reduced = bms_add_member(reduced,
+										 dep->attributes[dep->nattributes-1]);
+		}
+	}
+
+	return reduced;
+}
+
+/*
+ * reduce list of equality clauses using soft functional dependencies
+ *
+ * We simply walk through list of functional dependencies, and for each one we
+ * check whether the dependency 'matches' the clauses, i.e. if there's a clause
+ * matching the condition. If yes, we attempt to remove all clauses matching
+ * the implied part of the dependency from the list.
+ *
+ * This only reduces equality clauses, and ignores all the other types. We might
+ * extend it to handle IS NULL clause, in the future.
+ *
+ * We also assume the equality clauses are 'compatible'. For example we can't
+ * identify when the clauses use a mismatching zip code and city name. In such
+ * case the usual approach (product of selectivities) would produce a better
+ * estimate, although mostly by chance.
+ *
+ * The implementation needs to be careful about cyclic dependencies, e.g. when
+ *
+ *     (a -> b) and (b -> a)
+ *
+ * at the same time, which means there's 1:1 relationship between te columns.
+ * In this case we must not reduce clauses on both attributes at the same time.
+ *
+ * TODO Currently we only apply functional dependencies at the same level, but
+ *      maybe we could transfer the clauses from upper levels to the subtrees?
+ *      For example let's say we have (a->b) dependency, and condition
+ *
+ *          (a=1) AND (b=2 OR c=3)
+ *
+ *      Currently, we won't be able to perform any reduction, because we'll
+ *      consider (a=1) and (b=2 OR c=3) independently. But maybe we could pass
+ *      (a=1) into the other expression, and only check it against conditions
+ *      of the functional dependencies?
+ *
+ *      In this case we'd end up with
+ *
+ *         (a=1)
+ *
+ *      as we'd consider (b=2) implied thanks to the rule, rendering the whole
+ *      OR clause valid.
+ */
+static List *
+clauselist_apply_dependencies(PlannerInfo *root, List *clauses,
+							  Index relid, List *stats)
+{
+	Bitmapset  *clause_attnums = NULL;
+	Bitmapset  *reduced_attnums = NULL;
+
+	/*
+	 * Is there at least one statistics with functional dependencies?
+	 * If not, return the original clauses right away.
+	 *
+	 * XXX Isn't this a bit pointless, thanks to exactly the same check in
+	 *     clauselist_selectivity()? Can we trigger the condition here?
+	 */
+	if (! has_stats(stats, MV_CLAUSE_TYPE_FDEP))
+		return clauses;
+
+	/* collect attnums from clauses compatible with dependencies (equality) */
+	clause_attnums = collect_mv_attnums(clauses, relid);
+
+	/* decide which attnums may be eliminated */
+	reduced_attnums = fdeps_reduce_attnums(stats, clause_attnums);
+
+	/*
+	 * Walk through the clauses, and see which other clauses we may reduce.
+	 */
+	clauses = fdeps_reduce_clauses(clauses, relid, reduced_attnums);
+
+	bms_free(clause_attnums);
+	bms_free(reduced_attnums);
+
+	return clauses;
+}
+
+/*
+ * Check that there are stats with at least one of the requested types.
+ */
+static bool
+stats_type_matches(MVStatisticInfo *stat, int type)
+{
+	if ((type & MV_CLAUSE_TYPE_FDEP) && stat->deps_built)
+		return true;
+
+	return false;
+}
+
+/*
+ * Check that there are stats with at least one of the requested types.
+ */
+static bool
+has_stats(List *stats, int type)
+{
+	ListCell   *s;
+
+	foreach (s, stats)
+	{
+		MVStatisticInfo	*stat = (MVStatisticInfo *)lfirst(s);
+
+		/* terminate if we've found at least one matching statistics */
+		if (stats_type_matches(stat, type))
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * Lookups stats for a given baserel.
+ */
+static List *
+find_stats(PlannerInfo *root, Index relid)
+{
+	Assert(root->simple_rel_array[relid] != NULL);
+
+	return root->simple_rel_array[relid]->mvstatlist;
+}
diff --git a/src/backend/utils/mvstats/README.dependencies b/src/backend/utils/mvstats/README.dependencies
index 1f96fbc..f248459 100644
--- a/src/backend/utils/mvstats/README.dependencies
+++ b/src/backend/utils/mvstats/README.dependencies
@@ -156,37 +156,24 @@ estimates - especially compared to histograms, that are quite bad in estimating
 equality clauses.
 
 
-Limitations
------------
-
-Let's see the main liminations of functional dependencies, especially those
-related to the current implementation.
+Multi-column dependencies
+-------------------------
 
-The current implementation supports only dependencies between two columns, but
-this is merely a simplification of the initial implementation. It's certainly
-useful to mine for dependencies involving multiple columns on the 'left' side,
-i.e. a condition for the dependency. That is dependencies like (a,b -> c).
+The implementation supports dependencies with multiple columns on the left side
+(i.e. condition of the dependency). The detection starts from dependencies with
+a single condition, and then proceeds to higher condition counts.
 
-The implementation may/should be smart enough not to mine redundant conditions,
-e.g. (a->b) and (a,c -> b), because the latter is a trivial consequence of the
-former one (if values of 'a' determine 'b', adding another column won't change
-that relationship). The ANALYZE should first analyze 1:1 dependencies, then 2:1
-dependencies (and skip the already identified ones), etc.
+It also detects dependencies that are implied by already identified ones, and
+ignores them. For example if we know that (a->b) then we won't add (a,c->b) as
+this dependency is a trivial consequence of (a->b).
 
-For example the dependency
+For a more practical example, consider these two dependencies
 
     (city name -> zip code)
-
-is much stronger, i.e. whenever it hold, then
-
     (city name, state name -> zip code)
 
-holds too. But in case there are cities with the same name in different states,
-then only the latter dependency will be valid.
-
-Of course, there probably are cities with the same name within a single state,
-but hopefully this is relatively rare occurence (and thus we'll still detect
-the 'soft' dependency).
+We could say that the former dependency is stronger because if it's valid, then
+the second dependency is valid too.
 
 Handling multiple columns on the right side of the dependency, is not necessary,
 as those dependencies may be simply decomposed into a set of dependencies with
@@ -199,24 +186,22 @@ is exactly the same as
     (a -> b) & (a -> c)
 
 Of course, storing the first form may be more efficient thant storing multiple
-'simple' dependencies separately.
-
+'simple' dependencies separately. This is left as a future work.
 
-TODO Support dependencies with multiple columns on left/right.
 
-TODO Investigate using histogram and MCV list to verify the dependencies.
+Future work
+-----------
 
-TODO Investigate statistical testing of the distribution (to decide whether it
-     makes sense to build the histogram/MCV list).
+* Investigate using histogram and MCV list to verify the dependencies.
 
-TODO Using a min/max of selectivities would probably make more sense for the
-     associated columns.
+* Investigate statistical testing of the distribution (to decide whether it
+  makes sense to build the histogram/MCV list).
 
-TODO Consider eliminating the implied columns from the histogram and MCV lists
-     (but maybe that's not a good idea, because that'd make it impossible to use
-     these stats for non-equality clauses and also it wouldn't be possible to
-     use the stats for verification of the dependencies).
+* Consider eliminating the implied columns from the histogram and MCV lists
+  (but maybe that's not a good idea, because that'd make it impossible to use
+  these stats for non-equality clauses and also it wouldn't be possible to
+  use the stats for verification of the dependencies).
 
-TODO The reduction probably might be extended to also handle IS NULL clauses,
-     assuming we fix the ANALYZE to properly handle NULL values. We however
-     won't be able to reduce IS NOT NULL (unless I'm missing something).
+* The reduction probably might be extended to also handle IS NULL clauses,
+  assuming we fix the ANALYZE to properly handle NULL values. We however
+  won't be able to reduce IS NOT NULL (unless I'm missing something).
diff --git a/src/backend/utils/mvstats/README.stats b/src/backend/utils/mvstats/README.stats
new file mode 100644
index 0000000..a38ea7b
--- /dev/null
+++ b/src/backend/utils/mvstats/README.stats
@@ -0,0 +1,36 @@
+Multivariate statististics
+==========================
+
+When estimating various quantities (e.g. condition selectivities) the default
+approach relies on the assumption of independence. In practice that's often
+not true, resulting in estimation errors.
+
+Multivariate stats track different types of dependencies between the columns,
+hopefully improving the estimates.
+
+Currently we only have one kind of multivariate statistics - soft functional
+dependencies, and we use it to improve estimates of equality clauses. See
+README.dependencies for details.
+
+
+Selectivity estimation
+----------------------
+
+When estimating selectivity, we aim to achieve several things:
+
+    (a) maximize the estimate accuracy
+
+    (b) minimize the overhead, especially when no suitable multivariate stats
+        exist (so if you are not using multivariate stats, there's no overhead)
+
+This clauselist_selectivity() performs several inexpensive checks first, before
+even attempting to do the more expensive estimation.
+
+    (1) check if there are multivariate stats on the relation
+
+    (2) check there are at least two attributes referenced by clauses compatible
+        with multivariate statistics (equality clauses for func. dependencies)
+
+    (3) perform reduction of equality clauses using func. dependencies
+
+    (4) estimate the reduced list of clauses using regular statistics
diff --git a/src/backend/utils/mvstats/common.c b/src/backend/utils/mvstats/common.c
index 82f2177..dcb7c78 100644
--- a/src/backend/utils/mvstats/common.c
+++ b/src/backend/utils/mvstats/common.c
@@ -84,7 +84,8 @@ build_mv_stats(Relation onerel, int numrows, HeapTuple *rows,
 		/*
 		 * Analyze functional dependencies of columns.
 		 */
-		deps = build_mv_dependencies(numrows, rows, attrs, stats);
+		if (stat->deps_enabled)
+			deps = build_mv_dependencies(numrows, rows, attrs, stats);
 
 		/* store the histogram / MCV list in the catalog */
 		update_mv_stats(stat->mvoid, deps, attrs);
@@ -163,6 +164,7 @@ list_mv_stats(Oid relid)
 
 		info->mvoid = HeapTupleGetOid(htup);
 		info->stakeys = buildint2vector(stats->stakeys.values, stats->stakeys.dim1);
+		info->deps_enabled = stats->deps_enabled;
 		info->deps_built = stats->deps_built;
 
 		result = lappend(result, info);
@@ -274,6 +276,7 @@ compare_scalars_partition(const void *a, const void *b, void *arg)
 	return ApplySortComparator(da, false, db, false, ssup);
 }
 
+
 /* initialize multi-dimensional sort */
 MultiSortSupport
 multi_sort_init(int ndims)
diff --git a/src/backend/utils/mvstats/dependencies.c b/src/backend/utils/mvstats/dependencies.c
index 5437bdf..412dc30 100644
--- a/src/backend/utils/mvstats/dependencies.c
+++ b/src/backend/utils/mvstats/dependencies.c
@@ -684,3 +684,27 @@ pg_mv_stats_dependencies_show(PG_FUNCTION_ARGS)
 
 	PG_RETURN_TEXT_P(cstring_to_text(buf.data));
 }
+
+MVDependencies
+load_mv_dependencies(Oid mvoid)
+{
+	bool		isnull = false;
+	Datum		deps;
+
+	/* Prepare to scan pg_mv_statistic for entries having indrelid = this rel. */
+	HeapTuple	htup = SearchSysCache1(MVSTATOID, ObjectIdGetDatum(mvoid));
+
+#ifdef USE_ASSERT_CHECKING
+	Form_pg_mv_statistic	mvstat = (Form_pg_mv_statistic) GETSTRUCT(htup);
+	Assert(mvstat->deps_enabled && mvstat->deps_built);
+#endif
+
+	deps = SysCacheGetAttr(MVSTATOID, htup,
+						   Anum_pg_mv_statistic_stadeps, &isnull);
+
+	Assert(!isnull);
+
+	ReleaseSysCache(htup);
+
+	return deserialize_mv_dependencies(DatumGetByteaP(deps));
+}
diff --git a/src/include/utils/mvstats.h b/src/include/utils/mvstats.h
index 7837bc0..ec55a09 100644
--- a/src/include/utils/mvstats.h
+++ b/src/include/utils/mvstats.h
@@ -17,7 +17,6 @@
 #include "fmgr.h"
 #include "commands/vacuum.h"
 
-
 #define MVSTATS_MAX_DIMENSIONS	8		/* max number of attributes */
 
 /*
@@ -49,6 +48,8 @@ typedef MVDependenciesData* MVDependencies;
  *      stats specified using flags (or something like that).
  */
 
+MVDependencies load_mv_dependencies(Oid mvoid);
+
 bytea * serialize_mv_dependencies(MVDependencies dependencies);
 
 /* deserialization of stats (serialization is private to analyze) */
diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h
index 8771f9c..d09ba25 100644
--- a/src/include/utils/rel.h
+++ b/src/include/utils/rel.h
@@ -94,7 +94,7 @@ typedef struct RelationData
 	List	   *rd_indexlist;	/* list of OIDs of indexes on relation */
 	Oid			rd_oidindex;	/* OID of unique index on OID, if any */
 	Oid			rd_replidindex; /* OID of replica identity index, if any */
- 
+
 	/* data managed by RelationGetMVStatList: */
 	List	   *rd_mvstatlist;	/* list of OIDs of multivariate stats */
 
diff --git a/src/test/regress/expected/mv_dependencies.out b/src/test/regress/expected/mv_dependencies.out
index f54e1b7..ee8a9b2 100644
--- a/src/test/regress/expected/mv_dependencies.out
+++ b/src/test/regress/expected/mv_dependencies.out
@@ -58,8 +58,10 @@ SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
 
 TRUNCATE functional_dependencies;
 -- a => b, a => c, b => c
+-- check explain (expect bitmap index scan, not plain index scan)
 INSERT INTO functional_dependencies
      SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i);
+CREATE INDEX fdeps_idx ON functional_dependencies (a, b);
 ANALYZE functional_dependencies;
 SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
   FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass;
@@ -68,6 +70,16 @@ SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
  t            | t          | (1) => 2, (1) => 3, (2) => 3
 (1 row)
 
+EXPLAIN (COSTS off)
+ SELECT * FROM functional_dependencies WHERE a = 10 AND b = 5;
+                 QUERY PLAN                  
+---------------------------------------------
+ Bitmap Heap Scan on functional_dependencies
+   Recheck Cond: ((a = 10) AND (b = 5))
+   ->  Bitmap Index Scan on fdeps_idx
+         Index Cond: ((a = 10) AND (b = 5))
+(4 rows)
+
 DROP TABLE functional_dependencies;
 -- varlena type (text)
 CREATE TABLE functional_dependencies (
@@ -113,8 +125,10 @@ SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
 
 TRUNCATE functional_dependencies;
 -- a => b, a => c, b => c
+-- check explain (expect bitmap index scan, not plain index scan)
 INSERT INTO functional_dependencies
      SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i);
+CREATE INDEX fdeps_idx ON functional_dependencies (a, b);
 ANALYZE functional_dependencies;
 SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
   FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass;
@@ -123,6 +137,16 @@ SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
  t            | t          | (1) => 2, (1) => 3, (2) => 3
 (1 row)
 
+EXPLAIN (COSTS off)
+ SELECT * FROM functional_dependencies WHERE a = '10' AND b = '5';
+                         QUERY PLAN                         
+------------------------------------------------------------
+ Bitmap Heap Scan on functional_dependencies
+   Recheck Cond: ((a = '10'::text) AND (b = '5'::text))
+   ->  Bitmap Index Scan on fdeps_idx
+         Index Cond: ((a = '10'::text) AND (b = '5'::text))
+(4 rows)
+
 DROP TABLE functional_dependencies;
 -- NULL values (mix of int and text columns)
 CREATE TABLE functional_dependencies (
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index bec0316..4f2ffb8 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -110,3 +110,6 @@ test: event_trigger
 
 # run stats by itself because its delay may be insufficient under heavy load
 test: stats
+
+# run tests of multivariate stats
+test: mv_dependencies
diff --git a/src/test/regress/sql/mv_dependencies.sql b/src/test/regress/sql/mv_dependencies.sql
index 051633a..8ba72a4 100644
--- a/src/test/regress/sql/mv_dependencies.sql
+++ b/src/test/regress/sql/mv_dependencies.sql
@@ -56,13 +56,20 @@ SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
 TRUNCATE functional_dependencies;
 
 -- a => b, a => c, b => c
+-- check explain (expect bitmap index scan, not plain index scan)
 INSERT INTO functional_dependencies
      SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i);
+
+CREATE INDEX fdeps_idx ON functional_dependencies (a, b);
+
 ANALYZE functional_dependencies;
 
 SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
   FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass;
 
+EXPLAIN (COSTS off)
+ SELECT * FROM functional_dependencies WHERE a = 10 AND b = 5;
+
 DROP TABLE functional_dependencies;
 
 -- varlena type (text)
@@ -99,6 +106,7 @@ TRUNCATE functional_dependencies;
 -- a => b, a => c
 INSERT INTO functional_dependencies
      SELECT i/10, i/150, i/200 FROM generate_series(1,10000) s(i);
+
 ANALYZE functional_dependencies;
 
 SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
@@ -107,13 +115,20 @@ SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
 TRUNCATE functional_dependencies;
 
 -- a => b, a => c, b => c
+-- check explain (expect bitmap index scan, not plain index scan)
 INSERT INTO functional_dependencies
      SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i);
+
+CREATE INDEX fdeps_idx ON functional_dependencies (a, b);
+
 ANALYZE functional_dependencies;
 
 SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps)
   FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass;
 
+EXPLAIN (COSTS off)
+ SELECT * FROM functional_dependencies WHERE a = '10' AND b = '5';
+
 DROP TABLE functional_dependencies;
 
 -- NULL values (mix of int and text columns)
-- 
2.5.0

