From c118bc5a249f58733f8bb6e0eaaf430e71754c10 Mon Sep 17 00:00:00 2001 From: amit Date: Thu, 18 Jul 2019 10:22:31 +0900 Subject: [PATCH v3 1/3] Some cosmetic improvements to partitionwise join code Among other changes, this moves a couple of functions from joinrel.c to relnode.c. --- src/backend/optimizer/path/joinrels.c | 167 --------------------- src/backend/optimizer/util/plancat.c | 20 +-- src/backend/optimizer/util/relnode.c | 268 +++++++++++++++++++++++++++++----- src/include/nodes/pathnodes.h | 36 +++-- src/include/optimizer/paths.h | 3 - 5 files changed, 271 insertions(+), 223 deletions(-) diff --git a/src/backend/optimizer/path/joinrels.c b/src/backend/optimizer/path/joinrels.c index 6a480ab764..fa68059c3f 100644 --- a/src/backend/optimizer/path/joinrels.c +++ b/src/backend/optimizer/path/joinrels.c @@ -46,8 +46,6 @@ static void try_partitionwise_join(PlannerInfo *root, RelOptInfo *rel1, static SpecialJoinInfo *build_child_join_sjinfo(PlannerInfo *root, SpecialJoinInfo *parent_sjinfo, Relids left_relids, Relids right_relids); -static int match_expr_to_partition_keys(Expr *expr, RelOptInfo *rel, - bool strict_op); /* @@ -1573,168 +1571,3 @@ build_child_join_sjinfo(PlannerInfo *root, SpecialJoinInfo *parent_sjinfo, return sjinfo; } - -/* - * Returns true if there exists an equi-join condition for each pair of - * partition keys from given relations being joined. - */ -bool -have_partkey_equi_join(RelOptInfo *joinrel, - RelOptInfo *rel1, RelOptInfo *rel2, - JoinType jointype, List *restrictlist) -{ - PartitionScheme part_scheme = rel1->part_scheme; - ListCell *lc; - int cnt_pks; - bool pk_has_clause[PARTITION_MAX_KEYS]; - bool strict_op; - - /* - * This function should be called when the joining relations have same - * partitioning scheme. - */ - Assert(rel1->part_scheme == rel2->part_scheme); - Assert(part_scheme); - - memset(pk_has_clause, 0, sizeof(pk_has_clause)); - foreach(lc, restrictlist) - { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); - OpExpr *opexpr; - Expr *expr1; - Expr *expr2; - int ipk1; - int ipk2; - - /* If processing an outer join, only use its own join clauses. */ - if (IS_OUTER_JOIN(jointype) && - RINFO_IS_PUSHED_DOWN(rinfo, joinrel->relids)) - continue; - - /* Skip clauses which can not be used for a join. */ - if (!rinfo->can_join) - continue; - - /* Skip clauses which are not equality conditions. */ - if (!rinfo->mergeopfamilies && !OidIsValid(rinfo->hashjoinoperator)) - continue; - - opexpr = castNode(OpExpr, rinfo->clause); - - /* - * The equi-join between partition keys is strict if equi-join between - * at least one partition key is using a strict operator. See - * explanation about outer join reordering identity 3 in - * optimizer/README - */ - strict_op = op_strict(opexpr->opno); - - /* Match the operands to the relation. */ - if (bms_is_subset(rinfo->left_relids, rel1->relids) && - bms_is_subset(rinfo->right_relids, rel2->relids)) - { - expr1 = linitial(opexpr->args); - expr2 = lsecond(opexpr->args); - } - else if (bms_is_subset(rinfo->left_relids, rel2->relids) && - bms_is_subset(rinfo->right_relids, rel1->relids)) - { - expr1 = lsecond(opexpr->args); - expr2 = linitial(opexpr->args); - } - else - continue; - - /* - * Only clauses referencing the partition keys are useful for - * partitionwise join. - */ - ipk1 = match_expr_to_partition_keys(expr1, rel1, strict_op); - if (ipk1 < 0) - continue; - ipk2 = match_expr_to_partition_keys(expr2, rel2, strict_op); - if (ipk2 < 0) - continue; - - /* - * If the clause refers to keys at different ordinal positions, it can - * not be used for partitionwise join. - */ - if (ipk1 != ipk2) - continue; - - /* - * The clause allows partitionwise join if only it uses the same - * operator family as that specified by the partition key. - */ - if (rel1->part_scheme->strategy == PARTITION_STRATEGY_HASH) - { - if (!op_in_opfamily(rinfo->hashjoinoperator, - part_scheme->partopfamily[ipk1])) - continue; - } - else if (!list_member_oid(rinfo->mergeopfamilies, - part_scheme->partopfamily[ipk1])) - continue; - - /* Mark the partition key as having an equi-join clause. */ - pk_has_clause[ipk1] = true; - } - - /* Check whether every partition key has an equi-join condition. */ - for (cnt_pks = 0; cnt_pks < part_scheme->partnatts; cnt_pks++) - { - if (!pk_has_clause[cnt_pks]) - return false; - } - - return true; -} - -/* - * Find the partition key from the given relation matching the given - * expression. If found, return the index of the partition key, else return -1. - */ -static int -match_expr_to_partition_keys(Expr *expr, RelOptInfo *rel, bool strict_op) -{ - int cnt; - - /* This function should be called only for partitioned relations. */ - Assert(rel->part_scheme); - - /* Remove any relabel decorations. */ - while (IsA(expr, RelabelType)) - expr = (Expr *) (castNode(RelabelType, expr))->arg; - - for (cnt = 0; cnt < rel->part_scheme->partnatts; cnt++) - { - ListCell *lc; - - Assert(rel->partexprs); - foreach(lc, rel->partexprs[cnt]) - { - if (equal(lfirst(lc), expr)) - return cnt; - } - - if (!strict_op) - continue; - - /* - * If it's a strict equi-join a NULL partition key on one side will - * not join a NULL partition key on the other side. So, rows with NULL - * partition key from a partition on one side can not join with those - * from a non-matching partition on the other side. So, search the - * nullable partition keys as well. - */ - Assert(rel->nullable_partexprs); - foreach(lc, rel->nullable_partexprs[cnt]) - { - if (equal(lfirst(lc), expr)) - return cnt; - } - } - - return -1; -} diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index cf1761401d..7f4195e061 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -2248,9 +2248,8 @@ find_partition_scheme(PlannerInfo *root, Relation relation) /* * set_baserel_partition_key_exprs * - * Builds partition key expressions for the given base relation and sets them - * in given RelOptInfo. Any single column partition keys are converted to Var - * nodes. All Var nodes are restamped with the relid of given relation. + * Builds partition key expressions for the given base relation and sets + * rel->partexprs. */ static void set_baserel_partition_key_exprs(Relation relation, @@ -2298,17 +2297,20 @@ set_baserel_partition_key_exprs(Relation relation, lc = lnext(partkey->partexprs, lc); } + /* Base relations have a single expression per key. */ partexprs[cnt] = list_make1(partexpr); } + /* + * For base relations, we assume that the partition keys are non-nullable, + * although they are nullable in principle; list and hash partitioned + * tables may contain nulls in the partition key(s), for example. + * Assuming non-nullability is okay for the considerations of partition + * pruning, because pruning is never performed with non-strict operators. + */ rel->partexprs = partexprs; - /* - * A base relation can not have nullable partition key expressions. We - * still allocate array of empty expressions lists to keep partition key - * expression handling code simple. See build_joinrel_partition_info() and - * match_expr_to_partition_keys(). - */ + /* Assigning NIL for each key means there are no nullable keys. */ rel->nullable_partexprs = (List **) palloc0(sizeof(List *) * partnatts); } diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 85415381fb..d5b5baaf2e 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -29,6 +29,7 @@ #include "optimizer/tlist.h" #include "partitioning/partbounds.h" #include "utils/hsearch.h" +#include "utils/lsyscache.h" typedef struct JoinHashEntry @@ -58,6 +59,14 @@ static void add_join_rel(PlannerInfo *root, RelOptInfo *joinrel); static void build_joinrel_partition_info(RelOptInfo *joinrel, RelOptInfo *outer_rel, RelOptInfo *inner_rel, List *restrictlist, JoinType jointype); +static bool have_partkey_equi_join(RelOptInfo *joinrel, + RelOptInfo *rel1, RelOptInfo *rel2, + JoinType jointype, List *restrictlist); +static int match_join_arg_to_partition_keys(Expr *expr, RelOptInfo *rel, + bool strict_op); +static void set_joinrel_partition_key_exprs(RelOptInfo *joinrel, + RelOptInfo *outer_rel, RelOptInfo *inner_rel, + JoinType jointype); static void build_child_join_reltarget(PlannerInfo *root, RelOptInfo *parentrel, RelOptInfo *childrel, @@ -1594,18 +1603,18 @@ find_param_path_info(RelOptInfo *rel, Relids required_outer) /* * build_joinrel_partition_info - * If the two relations have same partitioning scheme, their join may be - * partitioned and will follow the same partitioning scheme as the joining - * relations. Set the partition scheme and partition key expressions in - * the join relation. + * Checks if the two relations being joined can use partitionwise join + * and if yes, initialize partitioning information of the resulting + * partitioned relation + * + * This will set part_scheme and partition key expressions (partexprs and + * nullable_partexprs) if required. */ static void build_joinrel_partition_info(RelOptInfo *joinrel, RelOptInfo *outer_rel, RelOptInfo *inner_rel, List *restrictlist, JoinType jointype) { - int partnatts; - int cnt; PartitionScheme part_scheme; /* Nothing to do if partitionwise join technique is disabled. */ @@ -1672,11 +1681,8 @@ build_joinrel_partition_info(RelOptInfo *joinrel, RelOptInfo *outer_rel, */ joinrel->part_scheme = part_scheme; joinrel->boundinfo = outer_rel->boundinfo; - partnatts = joinrel->part_scheme->partnatts; - joinrel->partexprs = (List **) palloc0(sizeof(List *) * partnatts); - joinrel->nullable_partexprs = - (List **) palloc0(sizeof(List *) * partnatts); joinrel->nparts = outer_rel->nparts; + set_joinrel_partition_key_exprs(joinrel, outer_rel, inner_rel, jointype); joinrel->part_rels = (RelOptInfo **) palloc0(sizeof(RelOptInfo *) * joinrel->nparts); @@ -1686,32 +1692,201 @@ build_joinrel_partition_info(RelOptInfo *joinrel, RelOptInfo *outer_rel, Assert(outer_rel->consider_partitionwise_join); Assert(inner_rel->consider_partitionwise_join); joinrel->consider_partitionwise_join = true; +} + +/* + * have_partkey_equi_join + * Returns true if there exist equi-join conditions involving pairs + * of matching partition keys of the relations being joined for all + * partition keys + */ +static bool +have_partkey_equi_join(RelOptInfo *joinrel, + RelOptInfo *rel1, RelOptInfo *rel2, + JoinType jointype, List *restrictlist) +{ + PartitionScheme part_scheme = rel1->part_scheme; + ListCell *lc; + int cnt_pks; + bool pk_has_clause[PARTITION_MAX_KEYS]; + bool strict_op; /* - * Construct partition keys for the join. - * - * An INNER join between two partitioned relations can be regarded as - * partitioned by either key expression. For example, A INNER JOIN B ON - * A.a = B.b can be regarded as partitioned on A.a or on B.b; they are - * equivalent. - * - * For a SEMI or ANTI join, the result can only be regarded as being - * partitioned in the same manner as the outer side, since the inner - * columns are not retained. - * - * An OUTER join like (A LEFT JOIN B ON A.a = B.b) may produce rows with - * B.b NULL. These rows may not fit the partitioning conditions imposed on - * B.b. Hence, strictly speaking, the join is not partitioned by B.b and - * thus partition keys of an OUTER join should include partition key - * expressions from the OUTER side only. However, because all - * commonly-used comparison operators are strict, the presence of nulls on - * the outer side doesn't cause any problem; they can't match anything at - * future join levels anyway. Therefore, we track two sets of - * expressions: those that authentically partition the relation - * (partexprs) and those that partition the relation with the exception - * that extra nulls may be present (nullable_partexprs). When the - * comparison operator is strict, the latter is just as good as the - * former. + * This function should be called when the joining relations have same + * partitioning scheme. + */ + Assert(rel1->part_scheme == rel2->part_scheme); + Assert(part_scheme); + + memset(pk_has_clause, 0, sizeof(pk_has_clause)); + foreach(lc, restrictlist) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + OpExpr *opexpr; + Expr *expr1; + Expr *expr2; + int ipk1; + int ipk2; + + /* If processing an outer join, only use its own join clauses. */ + if (IS_OUTER_JOIN(jointype) && + RINFO_IS_PUSHED_DOWN(rinfo, joinrel->relids)) + continue; + + /* Skip clauses which can not be used for a join. */ + if (!rinfo->can_join) + continue; + + /* Skip clauses which are not equality conditions. */ + if (!rinfo->mergeopfamilies && !OidIsValid(rinfo->hashjoinoperator)) + continue; + + opexpr = castNode(OpExpr, rinfo->clause); + + /* + * The equi-join between partition keys is strict if equi-join between + * at least one partition key is using a strict operator. See + * explanation about outer join reordering identity 3 in + * optimizer/README + */ + strict_op = op_strict(opexpr->opno); + + /* Match the operands to the relation. */ + if (bms_is_subset(rinfo->left_relids, rel1->relids) && + bms_is_subset(rinfo->right_relids, rel2->relids)) + { + expr1 = linitial(opexpr->args); + expr2 = lsecond(opexpr->args); + } + else if (bms_is_subset(rinfo->left_relids, rel2->relids) && + bms_is_subset(rinfo->right_relids, rel1->relids)) + { + expr1 = lsecond(opexpr->args); + expr2 = linitial(opexpr->args); + } + else + continue; + + /* + * Only clauses referencing the partition keys are useful for + * partitionwise join. + */ + ipk1 = match_join_arg_to_partition_keys(expr1, rel1, strict_op); + if (ipk1 < 0) + continue; + ipk2 = match_join_arg_to_partition_keys(expr2, rel2, strict_op); + if (ipk2 < 0) + continue; + + /* + * If the clause refers to keys at different ordinal positions, it can + * not be used for partitionwise join. + */ + if (ipk1 != ipk2) + continue; + + /* + * The clause allows partitionwise join if only it uses the same + * operator family as that specified by the partition key. + */ + if (rel1->part_scheme->strategy == PARTITION_STRATEGY_HASH) + { + if (!op_in_opfamily(rinfo->hashjoinoperator, + part_scheme->partopfamily[ipk1])) + continue; + } + else if (!list_member_oid(rinfo->mergeopfamilies, + part_scheme->partopfamily[ipk1])) + continue; + + /* Mark the partition key as having an equi-join clause. */ + pk_has_clause[ipk1] = true; + } + + /* Check whether every partition key has an equi-join condition. */ + for (cnt_pks = 0; cnt_pks < part_scheme->partnatts; cnt_pks++) + { + if (!pk_has_clause[cnt_pks]) + return false; + } + + return true; +} + +/* + * match_join_arg_to_partition_keys + * Tries to match a join clause argument expression to one of the nullable + * or non-nullable partition keys and if a match is found, returns the + * matched key's ordinal position or -1 if the expression could not be + * matched to any of the keys + */ +static int +match_join_arg_to_partition_keys(Expr *expr, RelOptInfo *rel, bool strict_op) +{ + int cnt; + + /* This function should be called only for partitioned relations. */ + Assert(rel->part_scheme); + + /* Remove any relabel decorations. */ + while (IsA(expr, RelabelType)) + expr = (Expr *) (castNode(RelabelType, expr))->arg; + + for (cnt = 0; cnt < rel->part_scheme->partnatts; cnt++) + { + ListCell *lc; + + Assert(rel->partexprs); + foreach(lc, rel->partexprs[cnt]) + { + if (equal(lfirst(lc), expr)) + return cnt; + } + + if (!strict_op) + continue; + + /* + * If it's a strict equi-join a NULL partition key on one side will + * not join a NULL partition key on the other side. So, rows with NULL + * partition key from a partition on one side can not join with those + * from a non-matching partition on the other side. So, search the + * nullable partition keys as well. + */ + Assert(rel->nullable_partexprs); + foreach(lc, rel->nullable_partexprs[cnt]) + { + if (equal(lfirst(lc), expr)) + return cnt; + } + } + + return -1; +} + +/* + * set_joinrel_partition_key_exprs + * Initialize partition key expressions + */ +static void +set_joinrel_partition_key_exprs(RelOptInfo *joinrel, + RelOptInfo *outer_rel, RelOptInfo *inner_rel, + JoinType jointype) +{ + int partnatts; + int cnt; + + Assert(joinrel->part_scheme != NULL); + + partnatts = joinrel->part_scheme->partnatts; + joinrel->partexprs = (List **) palloc0(sizeof(List *) * partnatts); + joinrel->nullable_partexprs = + (List **) palloc0(sizeof(List *) * partnatts); + + /* + * Join type determines which partition keys are assumed by the resulting + * join relation. Note that these keys are to be considered when checking + * if any further joins involving this joinrel may be partitioned. */ for (cnt = 0; cnt < partnatts; cnt++) { @@ -1725,18 +1900,37 @@ build_joinrel_partition_info(RelOptInfo *joinrel, RelOptInfo *outer_rel, switch (jointype) { + /* + * Join relation resulting from an INNER join may be regarded as + * partitioned by either of inner and outer relation keys. For + * example, A INNER JOIN B ON A.a = B.b can be regarded as + * partitioned on either A.a or B.b. + */ case JOIN_INNER: partexpr = list_concat_copy(outer_expr, inner_expr); nullable_partexpr = list_concat_copy(outer_null_expr, inner_null_expr); break; + /* + * Join relation resulting from a SEMI or ANTI join may be + * regarded as partitioned on the outer relation keys, since the + * inner columns are omitted from the output. + */ case JOIN_SEMI: case JOIN_ANTI: partexpr = list_copy(outer_expr); nullable_partexpr = list_copy(outer_null_expr); break; + /* + * Join relation resulting from a LEFT OUTER JOIN likewise may be + * regarded as partitioned on the (non-nullable) outer relation + * keys. The nullability of inner relation keys prevents them to + * be considered partition keys of the join relation in all cases, + * but they are okay as partition keys for further joins that + * involve strict join operators. + */ case JOIN_LEFT: partexpr = list_copy(outer_expr); nullable_partexpr = list_concat_copy(inner_expr, @@ -1745,6 +1939,12 @@ build_joinrel_partition_info(RelOptInfo *joinrel, RelOptInfo *outer_rel, inner_null_expr); break; + /* + * For FULL OUTER JOINs, both relations are nullable, so the + * resulting join relation may be regarded as partitioned on + * either of inner and outer relation keys, but only for joins + * that involve strict join operators. + */ case JOIN_FULL: nullable_partexpr = list_concat_copy(outer_expr, inner_expr); diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 23a06d718e..80a5cb77f4 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -584,16 +584,32 @@ typedef struct PartitionSchemeData *PartitionScheme; * this relation that are partitioned tables * themselves, in hierarchical order * - * Note: A base relation always has only one set of partition keys, but a join - * relation may have as many sets of partition keys as the number of relations - * being joined. partexprs and nullable_partexprs are arrays containing - * part_scheme->partnatts elements each. Each of these elements is a list of - * partition key expressions. For a base relation each list in partexprs - * contains only one expression and nullable_partexprs is not populated. For a - * join relation, partexprs and nullable_partexprs contain partition key - * expressions from non-nullable and nullable relations resp. Lists at any - * given position in those arrays together contain as many elements as the - * number of joining relations. + * Notes on partition key expressions (partexprs and nullable_partexprs): + * + * Partition key expressions will be used to spot references to the partition + * keys of the relation in the expressions of a given query so as to apply + * various partitioning-based optimizations to certain query constructs. For + * example, pruning unnecessary partitions of a table using baserestrictinfo + * clauses that contain partition keys, converting a join between two + * partitioned relations into a series of joins between pairs of their + * constituent partitions if the joined rows follow the same partitioning + * as the relations being joined. + * + * The partexprs and nullable_partexprs arrays each contain + * part_scheme->partnatts elements. Each of the elements is a list of + * partition key expressions. For partitioned *base* relations, there is one + * expression in every list, whereas for partitioned *join* relations, there + * can be as many as the number of component relations. + * + * nullable_partexprs are populated only in partitioned *join* relationss, + * that is, if any of their component relations are nullable due to OUTER JOIN + * considerations. It contains only the expressions of the nullable component + * relations, while those of the non-nullable relations are present in the + * partexprs. For the considerations of partitionwise join, nullable partition + * keys can be considered to partition the underlying relation in the same + * manner as the non-nullable partition keys do, as long as the join operator + * is stable, because those null-valued keys can't be joined further, thus + * preserving the partitioning. *---------- */ typedef enum RelOptKind diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 7345137d1d..54610b8656 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -106,9 +106,6 @@ extern bool have_join_order_restriction(PlannerInfo *root, extern bool have_dangerous_phv(PlannerInfo *root, Relids outer_relids, Relids inner_params); extern void mark_dummy_rel(RelOptInfo *rel); -extern bool have_partkey_equi_join(RelOptInfo *joinrel, - RelOptInfo *rel1, RelOptInfo *rel2, - JoinType jointype, List *restrictlist); /* * equivclass.c -- 2.11.0