From ad5f5b59265762b3674068ee3656baec3ec66b70 Mon Sep 17 00:00:00 2001
From: Alexander Kuzmenkov <a.kuzmenkov@postgrespro.ru>
Date: Wed, 13 Jun 2018 21:45:32 +0300
Subject: [PATCH 2/2] Remove unique self joins.

---
 src/backend/optimizer/plan/analyzejoins.c | 513 ++++++++++++++++++++++++++++++
 src/backend/optimizer/plan/planmain.c     |   5 +
 src/include/optimizer/planmain.h          |   1 +
 src/test/regress/expected/join.out        |  38 ++-
 src/test/regress/sql/join.sql             |  16 +
 5 files changed, 569 insertions(+), 4 deletions(-)

diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c
index c03010c..f593a17 100644
--- a/src/backend/optimizer/plan/analyzejoins.c
+++ b/src/backend/optimizer/plan/analyzejoins.c
@@ -22,12 +22,15 @@
  */
 #include "postgres.h"
 
+#include "catalog/pg_class.h"
 #include "nodes/nodeFuncs.h"
 #include "optimizer/clauses.h"
 #include "optimizer/joininfo.h"
 #include "optimizer/pathnode.h"
 #include "optimizer/paths.h"
 #include "optimizer/planmain.h"
+#include "optimizer/predtest.h"
+#include "optimizer/restrictinfo.h"
 #include "optimizer/tlist.h"
 #include "optimizer/var.h"
 #include "utils/lsyscache.h"
@@ -1122,3 +1125,513 @@ is_innerrel_unique_for(PlannerInfo *root,
 	/* Let rel_is_distinct_for() do the hard work */
 	return rel_is_distinct_for(root, innerrel, clause_list, unique_index);
 }
+
+static bool
+set_varno_walker(Node *node, void *context)
+{
+	if (node == NULL)
+		return false;
+
+	if (IsA(node, Var))
+		((Var *) node)->varno = (Index) (intptr_t) context;
+
+	return expression_tree_walker(node, set_varno_walker, context);
+}
+
+/*
+ * Replace varno with 'relid' for all Vars in the expression.
+ */
+static void
+set_varno(Expr *expr, Index relid)
+{
+	set_varno_walker((Node *) expr, (void*) (intptr_t) relid);
+}
+
+static bool
+references_relation_walker(Node *node, void *context)
+{
+	if (node == NULL)
+		return false;
+
+	if (IsA(node, Var))
+		return ((Var *) node)->varno == (Index) (intptr_t) context;
+
+	return expression_tree_walker(node, references_relation_walker, context);
+}
+
+/*
+ * Check whether the expression has any Vars from the relation identified
+ * by 'relid'. For EquivalenceClass nodes, checks the equivalence members.
+ */
+static bool
+references_relation(Node *node, Index relid)
+{
+	if (IsA(node, EquivalenceClass))
+	{
+		EquivalenceClass *ec = (EquivalenceClass *) node;
+		ListCell *lc;
+		foreach (lc, ec->ec_members)
+		{
+			if (references_relation(
+						(Node *) ((EquivalenceMember *) lfirst(lc))->em_expr,
+						relid))
+			{
+				return true;
+			}
+		}
+		return false;
+	}
+	return references_relation_walker(node, (void *) (intptr_t) relid);
+}
+
+/*
+ * Substitute newId for oldId in relids.
+ */
+static void
+change_relid(Relids *relids, Index oldId, Index newId)
+{
+	if (bms_is_member(oldId, *relids))
+		*relids = bms_add_member(bms_del_member(*relids, oldId), newId);
+}
+
+/*
+ * Remove a relation after we have proven that it participates only in an
+ * unneeded unique self join.
+ *
+ * The joinclauses list is destructively changed.
+ */
+static void
+remove_self_join_rel(PlannerInfo *root, Relids joinrelids, List *joinclauses,
+					 RelOptInfo *toKeep, RelOptInfo *toRemove)
+{
+	ListCell *prev, *cell, *next;
+	List *toAppend;
+
+	/*
+	 * Join clauses become restriction clauses, when Vars of the relation we
+	 * remove are replaced with corresponding Vars of the one we keep.
+	 * Restriction clauses from the removed relation are as well transferred
+	 * to the remaining one.
+	 */
+	toAppend = list_concat(joinclauses, toRemove->baserestrictinfo);
+
+	/*
+	 * Append the filters from the removed relation to the remaining one.
+	 */
+	foreach(cell, toAppend)
+	{
+		RestrictInfo *oldRinfo = lfirst_node(RestrictInfo, cell);
+		RestrictInfo *newRinfo = NULL;
+		Expr *newClause = NULL;
+
+		/*
+		 * Do not add multiple clauses derived from the same equivalence class.
+		 */
+		if (is_redundant_derived_clause(oldRinfo, toKeep->baserestrictinfo))
+			continue;
+
+		/*
+		 * Make a copy of the clause and replace the references to the removed
+		 * relation with references to the remaining one.
+		 */
+		newClause = (Expr *) copyObject(oldRinfo->clause);
+		set_varno(newClause, toKeep->relid);
+
+		/*
+		 * After we have replaced the Vars, check that the resulting clause is
+		 * not implied by the existing ones.
+		 */
+		if (!contain_mutable_functions((Node *) newClause)
+			&& predicate_implied_by(list_make1(newClause), toKeep->baserestrictinfo,
+								 /* weak = */ false ))
+			continue;			/* provably implied by r1 */
+
+		/*
+		 * If the clause has the form of "X=X", replace it with null test.
+		 */
+		if (oldRinfo->mergeopfamilies)
+		{
+			Assert(is_opclause(newClause));
+			Expr *leftOp = (Expr *) get_leftop(newClause);
+			Expr *rightOp = (Expr *) get_rightop(newClause);
+			if (leftOp != NULL && equal(leftOp, rightOp))
+			{
+				NullTest *test = makeNode(NullTest);
+				test->arg = leftOp;
+				test->nulltesttype = IS_NOT_NULL;
+				test->argisrow = false;
+				test->location = -1;
+				newClause = (Expr *) test;
+			}
+		}
+
+		/*
+		 * Finally, correct the relids of the old rinfo, and replace the clause
+		 * with the one we just constructed, and append it to the remaining
+		 * relation.
+		 */
+		newRinfo = copyObject(oldRinfo);
+		newRinfo->clause = newClause;
+		change_relid(&newRinfo->required_relids, toRemove->relid, toKeep->relid);
+		change_relid(&newRinfo->left_relids, toRemove->relid, toKeep->relid);
+		change_relid(&newRinfo->right_relids, toRemove->relid, toKeep->relid);
+		change_relid(&newRinfo->clause_relids, toRemove->relid, toKeep->relid);
+		toKeep->baserestrictinfo = lappend(toKeep->baserestrictinfo, newRinfo);
+	}
+
+	/*
+	 * Remove the relation from the planner data structures.
+	 */
+	remove_rel_from_query(root, toRemove->relid, joinrelids);
+
+	/*
+	 * The equivalence classes that reference the removed rel can't also
+	 * reference anything other than the remaining rel, or else this
+	 * optimization wouldn't work (see rel_used_above_join).
+	 * 
+	 * This means we don't have any ECs that can generate join clauses,
+	 * and only have the ECs that can generate restrictions. But the restrictions
+	 * have been generated already, and we've just transferred them to the
+	 * remaining relation from join clauses and from the restrictions of the 
+	 * removed relation.
+	 * 
+	 * Therefore, we don't need anymore the ECs that reference the removed
+	 * relation, and can delete them.
+	 */
+	prev = NULL;
+	cell = NULL;
+	next = list_head(root->eq_classes);
+	while (next)
+	{
+		prev = cell;
+		cell = next;
+		next = lnext(next);
+
+		if (references_relation(lfirst(cell), toRemove->relid))
+		{
+			root->eq_classes = list_delete_cell(root->eq_classes, cell, prev);
+			cell = prev;
+		}
+	}
+}
+
+/*
+ * Test whether the relations are joined on the same unique column.
+ */
+static bool
+is_unique_self_join(PlannerInfo *root, Relids joinrelids, RelOptInfo *outer,
+					RelOptInfo *inner, List *restrictlist)
+{
+	IndexOptInfo *outeridx = NULL;
+	IndexOptInfo *inneridx = NULL;
+
+	innerrel_is_unique(root, joinrelids, inner->relids,
+						   outer, JOIN_INNER, restrictlist, true, &outeridx);
+	if (!outeridx)
+		return false;
+
+	innerrel_is_unique(root, joinrelids, outer->relids,
+						   inner, JOIN_INNER, restrictlist, true, &inneridx);
+	if (!inneridx)
+		return false;
+
+	/* We must have the same unique index for both relations. */
+	if (outeridx->indexoid != inneridx->indexoid)
+		return false;
+
+	/* A sanity check: this is the same index on the same relation. */
+	Assert(root->simple_rte_array[outer->relid]->relid
+			== root->simple_rte_array[inner->relid]->relid);
+
+	return true;
+}
+
+/*
+ * Scratch space for the unique self join removal code.
+ */
+typedef struct
+{
+	/* Temporary array for relation ids. */
+	Index *relids;
+
+	/*
+	 * Array of Relids, one for each relation, indexed by relation id.
+	 * Each element is a set of relation ids with which this relation
+	 * has a special join.
+	 */
+	Relids *special_join_rels;
+
+	/* Bitmapset for join relids that is used to avoid reallocation. */
+	Relids joinrelids;
+} UsjScratch;
+
+/*
+ * Find and remove unique self joins in a group of base relations that have
+ * the same Oid.
+ *
+ * Returns IntList of the relids that were removed.
+ */
+static List *
+remove_self_joins_one_group(PlannerInfo *root, Index *relids, int n, UsjScratch *scratch)
+{
+	Relids joinrelids = scratch->joinrelids;
+	List *result = NIL;
+	int i, o;
+
+	if (n < 2)
+		return NIL;
+
+	for (o = 0; o < n; o++)
+	{
+		RelOptInfo *outer;
+
+		if (relids[o] == 0)
+			/* Already removed. */
+			continue;
+
+		outer = root->simple_rel_array[relids[o]];
+		for (i = o + 1; i < n; i++)
+		{
+			RelOptInfo *inner;
+			List *restrictlist;
+
+			if (relids[i] == 0)
+				/* Already removed. */
+				continue;
+
+			inner = root->simple_rel_array[relids[i]];
+
+			/*
+			 * Unique semi and left joins are processed by reduce_unique_semijoins
+			 * and remove_useless_left_joins respectively, so we don't have to
+			 * deal with them here. For full and anti joins, this optimization does
+			 * not apply. Therefore, we do nothing if these relations have
+			 * a special join.
+			 */
+			if (bms_is_member(relids[i], scratch->special_join_rels[relids[o]]))
+				continue;
+
+			/* Reuse joinrelids bitset to avoid reallocation. */
+			joinrelids = bms_del_members(joinrelids, joinrelids);
+
+			/*
+			 * We only deal with base rels here, so their relids bitset
+			 * contains only one member -- their relid.
+			 */
+			joinrelids = bms_add_member(joinrelids, relids[o]);
+			joinrelids = bms_add_member(joinrelids, relids[i]);
+
+			/* Is it a unique self join? */
+			restrictlist = build_joinrel_restrictlist(root, joinrelids, outer,
+													  inner);
+			if (!is_unique_self_join(root, joinrelids, outer, inner,
+										   restrictlist))
+				continue;
+
+			/* A relation can be removed if it is not referenced above the join. */
+			if (rel_used_above_join(root, joinrelids, inner))
+			{
+				if (rel_used_above_join(root, joinrelids, outer))
+					/*
+					 * Both relations are referenced above the join, can't remove
+					 * either one.
+					 */
+					continue;
+				else
+				{
+					/* Remove outer. */
+					remove_self_join_rel(root, joinrelids, restrictlist,
+										 inner, outer);
+					result = lappend_int(result, relids[o]);
+					relids[o] = 0;
+					break;
+				}
+			}
+			else
+			{
+				/* Remove inner. */
+				remove_self_join_rel(root, joinrelids, restrictlist,
+									 outer, inner);
+				result = lappend_int(result, relids[i]);
+				relids[i] = 0;
+			}
+		}
+	}
+
+	scratch->joinrelids = joinrelids;
+	return result;
+}
+
+/*
+ * A qsort comparator to sort the relids by the relation Oid.
+ */
+static int
+compare_rte(const Index *left, const Index *right, PlannerInfo *root)
+{
+	return root->simple_rte_array[*left]->relid
+		< root->simple_rte_array[*right]->relid;
+}
+
+/*
+ * Find and remove unique self joins on a particular level of the join tree.
+ */
+static void
+remove_self_joins_one_level(PlannerInfo *root, List **joinlist, UsjScratch *scratch)
+{
+	ListCell *lc;
+	List *relidsToRemove = NIL;
+	Oid groupOid;
+	int groupStart;
+	int i;
+	int n = 0;
+	Index *relid_ascending = scratch->relids;
+
+	/*
+	 * Collect the ids of base relations at this level of the join tree.
+	 */
+	foreach (lc, *joinlist)
+	{
+		RangeTblRef *ref = (RangeTblRef *) lfirst(lc);
+		if (!IsA(ref, RangeTblRef))
+			continue;
+
+		/*
+		 * We only care about base relations from which we select something.
+		 */
+		if (root->simple_rte_array[ref->rtindex]->rtekind == RTE_RELATION
+			&& root->simple_rte_array[ref->rtindex]->relkind == RELKIND_RELATION
+			&& root->simple_rel_array[ref->rtindex] != NULL)
+		{
+			relid_ascending[n++] = ref->rtindex;
+		}
+
+		/*
+		 * Limit the number of joins we process to control the quadratic behavior.
+		 */
+		if (n > join_collapse_limit)
+			break;
+	}
+
+	if (n < 2)
+		return;
+
+	/*
+	 * Find and process the groups of relations that have same Oid.
+	 */
+	qsort_arg(relid_ascending, n, sizeof(*relid_ascending),
+			  (qsort_arg_comparator) compare_rte, root);
+	groupOid = root->simple_rte_array[relid_ascending[0]]->relid;
+	groupStart = 0;
+	for (i = 1; i < n; i++)
+	{
+		RangeTblEntry *rte = root->simple_rte_array[relid_ascending[i]];
+		Assert(rte->relid != InvalidOid);
+		if (rte->relid != groupOid)
+		{
+			relidsToRemove = list_concat(relidsToRemove,
+				remove_self_joins_one_group(root, &relid_ascending[groupStart],
+					i - groupStart, scratch));
+			groupOid = rte->relid;
+			groupStart = i;
+		}
+	}
+	Assert(groupOid != InvalidOid);
+	Assert(groupStart < n);
+	relidsToRemove = list_concat(relidsToRemove,
+		remove_self_joins_one_group(root, &relid_ascending[groupStart],
+			n - groupStart, scratch));
+
+	/*
+	 * Delete the removed relations from joinlist.
+	 */
+	foreach(lc, relidsToRemove)
+	{
+		Index indexToRemove = lfirst_int(lc);
+		ListCell *prev = NULL, *next = NULL;
+		ListCell *lc2 = list_head(*joinlist);
+		while (lc2)
+		{
+			next = lnext(lc2);
+			if (castNode(RangeTblRef, lfirst(lc2))->rtindex == indexToRemove)
+				*joinlist = list_delete_cell(*joinlist, lc2, prev);
+			else
+				prev = lc2;
+			lc2 = next;
+		}
+	}
+
+	return;
+}
+
+/*
+ * Find and remove unique self joins on a single level of a join tree, and
+ * recurse to handle deeper levels.
+ */
+static void
+remove_self_joins_recurse(PlannerInfo *root, List **joinlist, UsjScratch *scratch)
+{
+	ListCell *lc;
+	foreach (lc, *joinlist)
+	{
+		switch (((Node*) lfirst(lc))->type)
+		{
+			case T_List:
+				remove_self_joins_recurse(root, (List **) &lfirst(lc), scratch);
+				break;
+			case T_RangeTblRef:
+				break;
+			default:
+				Assert(false);
+		}
+	}
+	remove_self_joins_one_level(root, joinlist, scratch);
+}
+
+/*
+ * Find out which relations have special joins to which.
+ */
+static void
+find_special_joins(PlannerInfo *root, Relids *special_join_rels)
+{
+	ListCell *lc;
+	foreach(lc, root->join_info_list)
+	{
+		SpecialJoinInfo *info = (SpecialJoinInfo *) lfirst(lc);
+		int bit = -1;
+		while ((bit = bms_next_member(info->min_lefthand, bit)) >= 0)
+		{
+			RelOptInfo *rel = find_base_rel(root, bit);
+			special_join_rels[rel->relid] =
+				bms_add_members(special_join_rels[rel->relid], info->min_righthand);
+		}
+
+		bit = -1;
+		while ((bit = bms_next_member(info->min_righthand, bit)) >= 0)
+		{
+			RelOptInfo *rel = find_base_rel(root, bit);
+			special_join_rels[rel->relid] =
+				bms_add_members(special_join_rels[rel->relid], info->min_lefthand);
+		}
+	}
+}
+
+/*
+ * Find and remove unique self joins in the entire join tree.
+ *
+ * First, we cache some data that will be needed later.
+ * Then, for each jointree level, we group all the participating
+ * base relations by their relation Oid. For every pair of relations in a
+ * group, we try to remove the join they make.
+ */
+void
+remove_useless_self_joins(PlannerInfo *root, List **joinlist)
+{
+	UsjScratch scratch;
+
+	scratch.relids = palloc(root->simple_rel_array_size * sizeof(Index));
+	scratch.special_join_rels = palloc0(root->simple_rel_array_size * sizeof(Relids));
+	scratch.joinrelids = NULL;
+
+	find_special_joins(root, scratch.special_join_rels);
+	remove_self_joins_recurse(root, joinlist, &scratch);
+}
diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c
index 8e4abbe..12c3868 100644
--- a/src/backend/optimizer/plan/planmain.c
+++ b/src/backend/optimizer/plan/planmain.c
@@ -199,6 +199,11 @@ query_planner(PlannerInfo *root, List *tlist,
 	reduce_unique_semijoins(root);
 
 	/*
+	 * Remove self joins on a unique column.
+	 */
+	remove_useless_self_joins(root, &joinlist);
+
+	/*
 	 * Now distribute "placeholders" to base rels as needed.  This has to be
 	 * done after join removal because removal could change whether a
 	 * placeholder is evaluable at a base rel.
diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h
index 4805f74..501bb9a 100644
--- a/src/include/optimizer/planmain.h
+++ b/src/include/optimizer/planmain.h
@@ -111,6 +111,7 @@ extern bool innerrel_is_unique(PlannerInfo *root,
 				   Relids joinrelids, Relids outerrelids, RelOptInfo *innerrel,
 				   JoinType jointype, List *restrictlist, bool force_cache,
 							   IndexOptInfo **unique_index);
+extern void remove_useless_self_joins(PlannerInfo *root, List **jointree);
 
 /*
  * prototypes for plan/setrefs.c
diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out
index dc6262b..05e9403 100644
--- a/src/test/regress/expected/join.out
+++ b/src/test/regress/expected/join.out
@@ -4307,11 +4307,13 @@ explain (costs off)
 select p.* from
   (parent p left join child c on (p.k = c.k)) join parent x on p.k = x.k
   where p.k = 1 and p.k = 2;
-        QUERY PLAN        
---------------------------
+                   QUERY PLAN                   
+------------------------------------------------
  Result
-   One-Time Filter: false
-(2 rows)
+   One-Time Filter: (false AND false)
+   ->  Index Scan using parent_pkey on parent p
+         Index Cond: (k = 1)
+(4 rows)
 
 -- bug 5255: this is not optimizable by join removal
 begin;
@@ -4427,6 +4429,34 @@ select * from
 ----+----+----+----
 (0 rows)
 
+-- test that semi- or inner self-joins on a unique column are removed
+create table sj (a int unique, b int);
+insert into sj values (1, null), (null, 2), (2, 1);
+analyze sj;
+select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1;
+ a | b 
+---+---
+ 2 | 1
+(1 row)
+
+explain (costs off)
+select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1;
+                  QUERY PLAN                   
+-----------------------------------------------
+ Seq Scan on sj p
+   Filter: ((a IS NOT NULL) AND (b = (a - 1)))
+(2 rows)
+
+explain (costs off)
+select * from sj p
+where exists (select * from sj q
+				where q.a = p.a and q.b < 10);
+                QUERY PLAN                
+------------------------------------------
+ Seq Scan on sj p
+   Filter: ((a IS NOT NULL) AND (b < 10))
+(2 rows)
+
 --
 -- Test hints given on incorrect column references are useful
 --
diff --git a/src/test/regress/sql/join.sql b/src/test/regress/sql/join.sql
index d3ba2a1..7620981 100644
--- a/src/test/regress/sql/join.sql
+++ b/src/test/regress/sql/join.sql
@@ -1527,6 +1527,22 @@ select * from
 select * from
   int8_tbl x join (int4_tbl x cross join int4_tbl y(ff)) j on q1 = f1; -- ok
 
+-- test that semi- or inner self-joins on a unique column are removed
+
+create table sj (a int unique, b int);
+insert into sj values (1, null), (null, 2), (2, 1);
+analyze sj;
+
+select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1;
+
+explain (costs off)
+select p.* from sj p, sj q where q.a = p.a and q.b = q.a - 1;
+
+explain (costs off)
+select * from sj p
+where exists (select * from sj q
+				where q.a = p.a and q.b < 10);
+
 --
 -- Test hints given on incorrect column references are useful
 --
-- 
2.7.4

