From 0e03546e6a0107e3d1371a7887f661bbb9484f53 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Sat, 24 Feb 2018 00:21:40 +1300 Subject: [PATCH v12 3/3] Pre-process OR clauses and store in the PartitionClauseInfo Previously OR clauses were not matched up to the partition keys during the initial stages of partition elimination. The work of processing these was left until a later stage of the partition elimination function set. This might have been fine for performing partition elimination at planning time, but when performing at execution time it meant that the OR clauses had to be matched to the partition keys each time we had to redetermine the matching partitions. During execution this event occurs each time a parameter whose value could affect the matching partitions changes, so in cases like parameterized nested loops it could be every tuple. Not processing the OR clauses was also not of much use since we must know all parameter IDs matching partition keys in advance, so that we're properly able to determine which parameter changes we must redetermine the matching partitions again for. Quite possibly this refactor should be tweaked and made part of the faster partition pruning patch set. --- src/backend/executor/nodeAppend.c | 8 +- src/backend/optimizer/util/partprune.c | 174 +++++++++++++++++++-------------- src/include/optimizer/partprune.h | 11 ++- 3 files changed, 114 insertions(+), 79 deletions(-) diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 7171682..7658906 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -83,6 +83,7 @@ struct ParallelAppendState struct PartitionPruneContextCache { PartitionPruneContext *context; + PartitionClauseInfo *partclauseinfo; PartitionPruneContextCache *subcache; }; @@ -714,7 +715,7 @@ set_valid_runtime_subplans_recurse(AppendState *node, context->econtext = node->ps.ps_ExprContext; context->paramids = NULL; - generate_partition_clauses(context, pinfo->prunequal); + ctxcache->partclauseinfo = generate_partition_clauses(context, pinfo->prunequal); node->part_prune_params = bms_add_members(node->part_prune_params, context->paramids); @@ -730,7 +731,7 @@ set_valid_runtime_subplans_recurse(AppendState *node, * Detect if any impossibilities were discovered during * generate_partition_clauses */ - if (context->clauseinfo->constfalse) + if (ctxcache->partclauseinfo->constfalse) { bms_free(*validsubplans); *validsubplans = NULL; @@ -747,7 +748,8 @@ set_valid_runtime_subplans_recurse(AppendState *node, * params. */ if (!bms_is_empty(context->paramids)) - partset = get_partitions_from_clauses(context); + partset = get_partitions_from_clauses(context, + ctxcache->partclauseinfo); else partset = pinfo->allsubnodes; diff --git a/src/backend/optimizer/util/partprune.c b/src/backend/optimizer/util/partprune.c index 21e86ff..5b2ddad 100644 --- a/src/backend/optimizer/util/partprune.c +++ b/src/backend/optimizer/util/partprune.c @@ -24,21 +24,19 @@ * pruning from a list of clauses containing clauses that reference a given * partitioned table. For example, prune_append_rel_partitions() calls this * function, because a partitioned table's rel->baserestrictinfo may contain - * clauses that might be useful for partitioning. Caller must have set up a - * valid partition pruning context in the form of struct PartitionPruneContext, - * that is, each of its fields other other than clauseinfo must be valid before - * calling here. After extracting relevant clauses, clauseinfo is filled with - * information that will be used for actual pruning. + * clauses that might be useful for partitioning. The list of clauses is + * processed and a PartitionClauseInfo is returned which contains details of + * any clauses which could be matched to the partition keys of the relation + * defined in the context. * * get_partitions_from_clauses() * - * This is to be called to prune partitions based on relevant partitioning - * clauses. Caller must have called generate_partition_clauses() at least - * once and hence a valid partition pruning context must have already been - * created. Especially, PartitionPruneContext.clauseinfo must contain valid - * information. Partition pruning proceeds by extracting constant values - * from the clauses and comparing it with the partition bounds while also - * taking into account strategies of the operators in the matched clauses. + * This is to be called to prune partitions based on 'partclauseinfo'. Caller + * must have called generate_partition_clauses() in order to have generated + * a valid PartitionClauseInfo. Partition pruning proceeds by extracting + * constant values from the clauses and comparing it with the partition bounds + * while also taking into account strategies of the operators in the matched + * clauses. * * make_partition_pruneinfo() * @@ -104,18 +102,22 @@ typedef enum PartOpStrategy PART_OP_GREATER } PartOpStrategy; -static void extract_partition_clauses(PartitionPruneContext *context, - List *clauses); +static PartitionClauseInfo *extract_partition_clauses( + PartitionPruneContext *context, + List *clauses); static bool match_boolean_partition_clause(Expr *clause, Expr *partkey, Expr **rightop); static Bitmapset *get_partitions_from_or_args(PartitionPruneContext *context, - List *or_args); + List *or_args, + List *or_partclauselist); static void remove_redundant_clauses(PartitionPruneContext *context, + PartitionClauseInfo *partclauseinfo, List **minimalclauses); static bool partition_cmp_args(PartitionPruneContext *context, Oid parttypid, Oid partopfamily, PartClause *pc, PartClause *leftarg, PartClause *rightarg, bool *result); static bool extract_bounding_datums(PartitionPruneContext *context, + PartitionClauseInfo *clauseinfo, List **minimalclauses, PartScanKeyInfo *keys); static PartOpStrategy partition_op_strategy(char part_strategy, PartClause *pc, bool *incl); @@ -144,6 +146,7 @@ prune_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel) else { PartitionPruneContext context; + PartitionClauseInfo *partclauseinfo; int partnatts = rel->part_scheme->partnatts, i; @@ -168,13 +171,16 @@ prune_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel) context.econtext = NULL; context.paramids = NULL; - /* process clauses; context.clauseinfo will be set */ - generate_partition_clauses(&context, clauses); + /* process clauses */ + partclauseinfo = generate_partition_clauses(&context, clauses); - if (!context.clauseinfo->constfalse) + if (!partclauseinfo->constfalse) { /* Actual pruning happens here. */ - Bitmapset *partindexes = get_partitions_from_clauses(&context); + Bitmapset *partindexes; + + partindexes = get_partitions_from_clauses(&context, + partclauseinfo); /* Add selected partitions' RT indexes to result. */ i = -1; @@ -188,13 +194,10 @@ prune_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel) /* * generate_partition_clauses - * Analyzes clauses to find those that match the partition key and sets - * context->clauseinfo - * - * Ideally, this should be called only once for a given query and a given - * partitioned table. + * Processes 'clauses' and returns a PartitionClauseInfo which contains + * the details of any clauses which were matched to partition keys. */ -void +PartitionClauseInfo * generate_partition_clauses(PartitionPruneContext *context, List *clauses) { /* The clauses list may be modified below, so better make a copy. */ @@ -227,25 +230,26 @@ generate_partition_clauses(PartitionPruneContext *context, List *clauses) clauses = list_concat(clauses, partqual); } - /* And away we go to do the real work; context->clauseinfo will be set */ - extract_partition_clauses(context, clauses); + /* pre-process the clauses and generate the PartitionClauseInfo */ + return extract_partition_clauses(context, clauses); } /* * get_partitions_from_clauses * Determine partitions that could possible contain a record that - * satisfies clauses as described in context->clauseinfo + * satisfies clauses as described in partclauseinfo * * Returns a Bitmapset of the matching partition indexes, or NULL if none can * match. */ Bitmapset * -get_partitions_from_clauses(PartitionPruneContext *context) +get_partitions_from_clauses(PartitionPruneContext *context, + PartitionClauseInfo *partclauseinfo) { - PartitionClauseInfo *partclauseinfo = context->clauseinfo; PartScanKeyInfo keys; Bitmapset *result; - ListCell *lc; + ListCell *lc; + ListCell *lc2; Assert(partclauseinfo != NULL); Assert(!partclauseinfo->constfalse); @@ -260,17 +264,18 @@ get_partitions_from_clauses(PartitionPruneContext *context) List *minimalclauses[PARTITION_MAX_KEYS]; /* - * For each partition key column, populate its slot in minimalclauses - * with the most restrictive of the clauses from the corresponding - * list in context->clauseinfo. + * For each partition key column, populate its element in + * minimalclauses with the most restrictive set of the clauses from + * the corresponding partition key in partclauseinfo. */ - remove_redundant_clauses(context, minimalclauses); + remove_redundant_clauses(context, partclauseinfo, minimalclauses); /* Did remove_redundant_clauses find any contradicting clauses? */ if (partclauseinfo->constfalse) return NULL; - if (extract_bounding_datums(context, minimalclauses, &keys)) + if (extract_bounding_datums(context, partclauseinfo, minimalclauses, + &keys)) { result = get_partitions_for_keys(context, &keys); @@ -292,12 +297,14 @@ get_partitions_from_clauses(PartitionPruneContext *context) } /* Now apply the OR clauses. */ - foreach(lc, partclauseinfo->or_clauses) + forboth(lc, partclauseinfo->or_clauses, lc2, + partclauseinfo->or_partclauseinfos) { List *or_args = (List *) lfirst(lc); + List *or_partclauselist = lfirst(lc2); Bitmapset *or_parts; - or_parts = get_partitions_from_or_args(context, or_args); + or_parts = get_partitions_from_or_args(context, or_args, or_partclauselist); /* * Clauses in or_clauses are mutually conjunctive and also in @@ -327,26 +334,27 @@ get_partitions_from_clauses(PartitionPruneContext *context) /* * extract_partition_clauses * Processes 'clauses' to extract clause matching the partition key. - * This adds matched clauses to the list corresponding to particular key - * in context->clauseinfo. Also collects other useful clauses to assist - * in partition elimination, such as OR clauses, clauses containing <> - * operator, and IS [NOT] NULL clauses + * Returns a PartitionClauseInfo which stores the clauses which were + * matched to the partition key. The PartitionClauseInfo also collects + * other useful clauses to assist in partition elimination, such as OR + * clauses, clauses containing <> operator, and IS [NOT] NULL clauses * * We may also discover some contradiction in the clauses which means that no - * partition can possibly match. In this case, the function sets - * context->clauseinfo's 'constfalse' to true and exits immediately without - * processing any further clauses. In this case, the caller must be careful - * not to assume the context->clauseinfo is fully populated with all clauses. + * partition can possibly match. In this case, the function sets the + * returned PartitionClauseInfo's 'constfalse' to true and exits immediately + * without processing any further clauses. In this case, the caller must be + * careful not to assume the return value is fully populated with all clauses. */ -static void +static PartitionClauseInfo * extract_partition_clauses(PartitionPruneContext *context, List *clauses) { PartitionClauseInfo *partclauseinfo; ListCell *lc; - context->clauseinfo = partclauseinfo = palloc(sizeof(PartitionClauseInfo)); + partclauseinfo = palloc(sizeof(PartitionClauseInfo)); memset(partclauseinfo->keyclauses, 0, sizeof(partclauseinfo->keyclauses)); partclauseinfo->or_clauses = NIL; + partclauseinfo->or_partclauseinfos = NIL; partclauseinfo->ne_clauses = NIL; partclauseinfo->keyisnull = NULL; partclauseinfo->keyisnotnull = NULL; @@ -367,7 +375,7 @@ extract_partition_clauses(PartitionPruneContext *context, List *clauses) !DatumGetBool(((Const *) clause)->constvalue)) { partclauseinfo->constfalse = true; - return; + return partclauseinfo; } } @@ -558,7 +566,7 @@ extract_partition_clauses(PartitionPruneContext *context, List *clauses) if (bms_is_member(i, partclauseinfo->keyisnull)) { partclauseinfo->constfalse = true; - return; + return partclauseinfo; } /* Record that a strict clause has been seen for this key */ partclauseinfo->keyisnotnull = @@ -734,7 +742,7 @@ extract_partition_clauses(PartitionPruneContext *context, List *clauses) if (bms_is_member(i, partclauseinfo->keyisnotnull)) { partclauseinfo->constfalse = true; - return; + return partclauseinfo; } partclauseinfo->keyisnull = bms_add_member(partclauseinfo->keyisnull, @@ -746,7 +754,7 @@ extract_partition_clauses(PartitionPruneContext *context, List *clauses) if (bms_is_member(i, partclauseinfo->keyisnull)) { partclauseinfo->constfalse = true; - return; + return partclauseinfo; } partclauseinfo->keyisnotnull = @@ -760,6 +768,31 @@ extract_partition_clauses(PartitionPruneContext *context, List *clauses) partclauseinfo->foundkeyclauses = true; } } + + /* + * Now pre-process any OR clauses found above and generate + * PartitionClauseInfos for them. + */ + foreach(lc, partclauseinfo->or_clauses) + { + List *or_args = lfirst(lc); + List *pclauselist = NIL; + ListCell *lc2; + + foreach (lc2, or_args) + { + List *clauses = list_make1(lfirst(lc2)); + PartitionClauseInfo *orpartclauseinfo; + + orpartclauseinfo = extract_partition_clauses(context, clauses); + pclauselist = lappend(pclauselist, orpartclauseinfo); + } + + partclauseinfo->or_partclauseinfos = + lappend(partclauseinfo->or_partclauseinfos, pclauselist); + } + + return partclauseinfo; } /* @@ -829,10 +862,11 @@ match_boolean_partition_clause(Expr *clause, Expr *partkey, * clause in or_args. */ static Bitmapset * -get_partitions_from_or_args(PartitionPruneContext *context, List *or_args) +get_partitions_from_or_args(PartitionPruneContext *context, List *or_args, + List *or_partclauselist) { Bitmapset *result = NULL; - ListCell *lc; + ListCell *lc, *lc2; /* * When matching an OR expression, it is only checked if at least one of @@ -843,20 +877,13 @@ get_partitions_from_or_args(PartitionPruneContext *context, List *or_args) * clause refutes its partition constraint, that is, we can eliminate all * of its partitions. */ - foreach(lc, or_args) + forboth(lc, or_args, lc2, or_partclauselist) { List *clauses = list_make1(lfirst(lc)); - PartitionPruneContext subcontext; + PartitionClauseInfo *or_pclause = lfirst(lc2); Bitmapset *arg_partset; - /* - * All fields except clauseinfo are same as in the parent context, - * which will be set by calling extract_partition_clauses(). - */ - memcpy(&subcontext, context, sizeof(PartitionPruneContext)); - extract_partition_clauses(&subcontext, clauses); - - if (!subcontext.clauseinfo->foundkeyclauses) + if (!or_pclause->foundkeyclauses) { List *partconstr = context->partition_qual; @@ -873,8 +900,8 @@ get_partitions_from_or_args(PartitionPruneContext *context, List *or_args) return bms_add_range(NULL, 0, context->nparts - 1); } - if (!subcontext.clauseinfo->constfalse) - arg_partset = get_partitions_from_clauses(&subcontext); + if (!or_pclause->constfalse) + arg_partset = get_partitions_from_clauses(context, or_pclause); else arg_partset = NULL; @@ -887,8 +914,8 @@ get_partitions_from_or_args(PartitionPruneContext *context, List *or_args) /* * remove_redundant_clauses - * Processes the clauses contained in context->clauseinfo to remove the - * ones that are superseeded by other clauses which are more restrictive. + * Process 'partpruneinfo' to remove the clauses that are superseeded + * by other clauses which are more restrictive. * * Finished lists of clauses are returned in *minimalclauses which is an array * with one slot for each of the partition keys. @@ -900,16 +927,16 @@ get_partitions_from_or_args(PartitionPruneContext *context, List *or_args) * that the clauses cannot possibly match any partition. Impossible clauses * include things like: x = 1 AND x = 2, x > 0 and x < 10. The function * returns right after finding such a clause and before returning, sets - * constfalse in context->clauseinfo to inform the caller that we found such + * constfalse in 'partclauseinfo' to inform the caller that we found such * clause. */ static void remove_redundant_clauses(PartitionPruneContext *context, + PartitionClauseInfo *partclauseinfo, List **minimalclauses) { PartClause *hash_clause, *btree_clauses[BTMaxStrategyNumber]; - PartitionClauseInfo *partclauseinfo = context->clauseinfo; ListCell *lc; int s; int i; @@ -1220,8 +1247,9 @@ partition_cmp_args(PartitionPruneContext *context, Oid parttypid, /* * extract_bounding_datums - * Process clauses in context->clauseinfo and populate 'keys' with all - * min/max/equal/not-equal values that we're able to determine. + * Process 'clauseinfo' and populate 'keys' with all + * min/max/equal/not-equal values that we're able to + * determine. * * *minimalclauses is an array with partnatts members, each of which is a list * of the most restrictive clauses of each operator strategy for the given @@ -1237,9 +1265,9 @@ partition_cmp_args(PartitionPruneContext *context, Oid parttypid, */ static bool extract_bounding_datums(PartitionPruneContext *context, + PartitionClauseInfo *clauseinfo, List **minimalclauses, PartScanKeyInfo *keys) { - PartitionClauseInfo *clauseinfo = context->clauseinfo; bool need_next_eq, need_next_min, need_next_max; diff --git a/src/include/optimizer/partprune.h b/src/include/optimizer/partprune.h index a761e65..ff68d4a 100644 --- a/src/include/optimizer/partprune.h +++ b/src/include/optimizer/partprune.h @@ -39,6 +39,9 @@ typedef struct PartitionClauseInfo /* Each members is a List itself of a given OR clauses's arguments. */ List *or_clauses; + /* each OR clause processed into a PartitionClauseInfo */ + List *or_partclauseinfos; + /* List of clauses containing <> operator. */ List *ne_clauses; @@ -55,9 +58,11 @@ typedef struct PartitionClauseInfo extern Relids prune_append_rel_partitions(PlannerInfo *root, RelOptInfo *rel); -extern void generate_partition_clauses(PartitionPruneContext *context, - List *clauses); -extern Bitmapset *get_partitions_from_clauses(PartitionPruneContext *context); +extern PartitionClauseInfo *generate_partition_clauses( + PartitionPruneContext *context, + List *clauses); +extern Bitmapset *get_partitions_from_clauses(PartitionPruneContext *context, + PartitionClauseInfo *partclauseinfo); extern PartitionPruneInfo *make_partition_pruneinfo(PlannerInfo *root, RelOptInfo *rel, -- 1.9.5.msysgit.1