From 32c9deae63176e6705db8dcd4734ab678d4e8823 Mon Sep 17 00:00:00 2001 From: "dgrowley@gmail.com" Date: Sun, 1 Apr 2018 01:18:42 +1300 Subject: [PATCH v18 1/5] Provide infrastructure to allow partition pruning during execution The query planner supports eliminating partitions of a partitioned table during query planning. This has its limitations as it can only perform the elimination using clauses which can be evaluated during planning. Allowing this partition elimination to occur during execution allows the values of Params to be used for elimination too, thus opening the door for PREPAREd statements to have unneeded partitions pruned too. The infrastructure provided here permits the building of a data structure which is able to perform the translation of the matching partition IDs as is returned by the existing partition pruning code into the List index of a subpaths list, as exist in node types such as Append, MergeAppend and ModifyTable. This allows us to translate a list of clauses into a Bitmapset of all the subpath indexes which must be included to satisfy the clause list. This commit does not add support for any node types. Support for this will arrive in follow-up commits. --- src/backend/catalog/partition.c | 23 ++ src/backend/commands/explain.c | 51 ++-- src/backend/executor/execPartition.c | 466 +++++++++++++++++++++++++++++++++ src/backend/nodes/copyfuncs.c | 20 ++ src/backend/nodes/outfuncs.c | 27 ++ src/backend/nodes/readfuncs.c | 19 ++ src/backend/optimizer/util/partprune.c | 257 ++++++++++++++++++ src/include/catalog/partition.h | 13 + src/include/executor/execPartition.h | 77 ++++++ src/include/nodes/nodes.h | 1 + src/include/nodes/primnodes.h | 23 ++ src/include/optimizer/partprune.h | 4 + 12 files changed, 963 insertions(+), 18 deletions(-) diff --git a/src/backend/catalog/partition.c b/src/backend/catalog/partition.c index bcd282515a..3be53a9a83 100644 --- a/src/backend/catalog/partition.c +++ b/src/backend/catalog/partition.c @@ -2051,6 +2051,29 @@ partkey_datum_from_expr(PartitionPruneContext *context, *value = ((Const *) expr)->constvalue; return true; + case T_Param: + /* + * When being called from the executor we may be able to evaluate + * the Param's value. + */ + if (context->planstate && + bms_is_member(((Param *) expr)->paramid, context->safeparams)) + { + ExprState *exprstate; + bool isNull; + + exprstate = ExecInitExpr(expr, context->planstate); + + *value = ExecEvalExprSwitchContext(exprstate, + context->planstate->ps_ExprContext, + &isNull); + + if (isNull) + return false; + + return true; + + } default: break; } diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 8a58672a94..a3db51e660 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -118,8 +118,8 @@ static void ExplainModifyTarget(ModifyTable *plan, ExplainState *es); static void ExplainTargetRel(Plan *plan, Index rti, ExplainState *es); static void show_modifytable_info(ModifyTableState *mtstate, List *ancestors, ExplainState *es); -static void ExplainMemberNodes(List *plans, PlanState **planstates, - List *ancestors, ExplainState *es); +static void ExplainMemberNodes(PlanState **planstates, int nsubnodes, + int nplans, List *ancestors, ExplainState *es); static void ExplainSubPlans(List *plans, List *ancestors, const char *relationship, ExplainState *es); static void ExplainCustomChildren(CustomScanState *css, @@ -1808,28 +1808,33 @@ ExplainNode(PlanState *planstate, List *ancestors, switch (nodeTag(plan)) { case T_ModifyTable: - ExplainMemberNodes(((ModifyTable *) plan)->plans, - ((ModifyTableState *) planstate)->mt_plans, + ExplainMemberNodes(((ModifyTableState *) planstate)->mt_plans, + ((ModifyTableState *) planstate)->mt_nplans, + list_length(((ModifyTable *) plan)->plans), ancestors, es); break; case T_Append: - ExplainMemberNodes(((Append *) plan)->appendplans, - ((AppendState *) planstate)->appendplans, + ExplainMemberNodes(((AppendState *) planstate)->appendplans, + ((AppendState *) planstate)->as_nplans, + list_length(((Append *) plan)->appendplans), ancestors, es); break; case T_MergeAppend: - ExplainMemberNodes(((MergeAppend *) plan)->mergeplans, - ((MergeAppendState *) planstate)->mergeplans, + ExplainMemberNodes(((MergeAppendState *) planstate)->mergeplans, + ((MergeAppendState *) planstate)->ms_nplans, + list_length(((MergeAppend *) plan)->mergeplans), ancestors, es); break; case T_BitmapAnd: - ExplainMemberNodes(((BitmapAnd *) plan)->bitmapplans, - ((BitmapAndState *) planstate)->bitmapplans, + ExplainMemberNodes(((BitmapAndState *) planstate)->bitmapplans, + ((BitmapAndState *) planstate)->nplans, + list_length(((BitmapAnd *) plan)->bitmapplans), ancestors, es); break; case T_BitmapOr: - ExplainMemberNodes(((BitmapOr *) plan)->bitmapplans, - ((BitmapOrState *) planstate)->bitmapplans, + ExplainMemberNodes(((BitmapOrState *) planstate)->bitmapplans, + ((BitmapOrState *) planstate)->nplans, + list_length(((BitmapOr *) plan)->bitmapplans), ancestors, es); break; case T_SubqueryScan: @@ -3140,18 +3145,28 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, * * The ancestors list should already contain the immediate parent of these * plans. - * - * Note: we don't actually need to examine the Plan list members, but - * we need the list in order to determine the length of the PlanState array. +* +* nsubnodes indicates the number of items in the planstates array. +* nplans indicates the original number of subnodes in the Plan, some of these +* may have been pruned by the run-time pruning code. */ static void -ExplainMemberNodes(List *plans, PlanState **planstates, +ExplainMemberNodes(PlanState **planstates, int nsubnodes, int nplans, List *ancestors, ExplainState *es) { - int nplans = list_length(plans); int j; - for (j = 0; j < nplans; j++) + /* + * The number of subnodes being lower than the number of subplans that + * was specified in the plan means that some subnodes have been ignored + * per instruction for the partition pruning code during the executor + * initialization. To make this a bit less mysterious, we'll indicate + * here that this has happened. + */ + if (nsubnodes < nplans) + ExplainPropertyInteger("Subplans Pruned", NULL, nplans - nsubnodes, es); + + for (j = 0; j < nsubnodes; j++) ExplainNode(planstates[j], ancestors, "Member", NULL, es); } diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 9a13188649..7c4f56c319 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -39,6 +39,12 @@ static char *ExecBuildSlotPartitionKeyDescription(Relation rel, bool *isnull, int maxfieldlen); static List *adjust_partition_tlist(List *tlist, TupleConversionMap *map); +static void find_subplans_for_extparams_recurse( + PartitionedRelPruning *partrelprune, + Bitmapset **validsubplans); +static void find_subplans_for_allparams_recurse( + PartitionedRelPruning *partrelprune, + Bitmapset **validsubplans); /* @@ -1151,3 +1157,463 @@ adjust_partition_tlist(List *tlist, TupleConversionMap *map) return new_tlist; } + +/*------------------------------------------------------------------------- + * Run-Time Partition Pruning Support. + * + * The following series of functions exist to support the removal of unneeded + * subnodes for queries against partitioned tables. The supporting functions + * here are designed to work with any node type which supports an arbitrary + * number of subnodes, e.g. Append, MergeAppend. + * + * Normally this pruning work is performed by the query planner's partition + * pruning code, however, the planner is limited to only being able to prune + * away unneeded partitions using quals which compare the partition key to a + * value which is known to be Const during planning. To allow the same + * pruning to be performed for values which are only determined during + * execution, we must make an additional pruning attempt during execution. + * + * Here we support pruning using both external and exec Params. The main + * difference between these that we need to concern ourselves with is the + * time when the values of the Params are known. External Param values are + * known at any time of execution, including executor startup, but exec Param + * values are only known when the executor is running. + * + * For external Params we may be able to prune away unneeded partitions + * during executor startup. This has the added benefit of not having to + * initialize the unneeded subnodes at all. This is useful as it can save + * quite a bit of effort during executor startup. + * + * For exec Params, we must delay pruning until the executor is running. + * + * Functions: + * + * ExecSetupPartitionPruning: + * This must be called by nodes before any partition pruning is + * attempted. Normally executor startup is a good time. This function + * creates the PartitionPruning details which are required by each + * of the two pruning functions, details include information about + * how to map the partition index details which are returned by the + * planner's partition prune function into subnode indexes. + * + * ExecFindInitialMatchingSubPlans: + * Returns indexes of matching subnodes utilizing only external Params + * to eliminate subnodes. The function must only be called during + * executor startup for the given node before the subnodes themselves + * are initialized. Subnodes which are found not to match by this + * function must not be included in the node's list of subnodes as this + * function performs a remap of the partition index to subplan index map + * and the newly created map provides indexes only for subnodes which + * remain after calling this function. + * + * ExecFindMatchingSubPlans: + * Returns indexes of matching subnodes utilizing all Params to eliminate + * subnodes which can't possibly contain matching tuples. This function + * can only be called while the executor is running. + *------------------------------------------------------------------------- + */ + +/* + * ExecSetupPartitionPruning + * + * Setup the required data structure for calling ExecFindMatchingSubPlans. + * + * 'partitionpruneinfo' is a List of PartitionPruneInfos as generated by + * make_partition_pruneinfo. Here we build a PartitionPruneContext for each + * item in the List. These context can be re-used each time we re-evaulate + * which partitions match the pruning steps provided in each + * PartitionPruneInfo. + */ +PartitionPruning * +ExecSetupPartitionPruning(PlanState *planstate, List *partitionpruneinfo) +{ + PartitionedRelPruning *partrelprunes; + PartitionPruning *partprune; + ListCell *lc; + int i; + + Assert(partitionpruneinfo != NIL); + + partprune = (PartitionPruning *) palloc(sizeof(PartitionPruning)); + partrelprunes = (PartitionedRelPruning *) + palloc(sizeof(PartitionedRelPruning) * + list_length(partitionpruneinfo)); + + /* + * The first item in the array contains the details for the query's target + * partition, so record that as the root of the partition hierarchy. + */ + partprune->partrelpruning = partrelprunes; + partprune->npartrelpruning = list_length(partitionpruneinfo); + partprune->extparams = NULL; + partprune->execparams = NULL; + partprune->allparams = NULL; + + /* + * Create a sub memory context which we'll use when making calls to the + * query planner's function to determine which partitions will match. The + * planner is not too careful about freeing memory, so we'll ensure we + * call the function in this context to avoid any memory leaking in the + * executor's memory context. + */ + partprune->prune_context = AllocSetContextCreate(CurrentMemoryContext, + "Partition Prune", + ALLOCSET_DEFAULT_SIZES); + + i = 0; + foreach(lc, partitionpruneinfo) + { + PartitionPruneInfo *pinfo = (PartitionPruneInfo *) lfirst(lc); + PartitionedRelPruning *partrelprune = &partrelprunes[i]; + PartitionPruneContext *context = &partrelprune->context; + PartitionDesc partdesc; + Relation rel; + PartitionKey partkey; + int partnatts; + int j; + + partrelprune->allpartindexes = bms_copy(pinfo->allpartindexes); + partrelprune->nparts = pinfo->nparts; + partrelprune->subnodeindex = palloc(sizeof(int) * pinfo->nparts); + partrelprune->subpartprune = palloc(sizeof(PartitionedRelPruning *) * + pinfo->nparts); + + /* + * We must make a copy of this rather than pointing directly to the + * plan's version as we may end up making modifications to it later. + */ + memcpy(partrelprune->subnodeindex, pinfo->subnodeindex, + sizeof(int) * pinfo->nparts); + + for (j = 0; j < pinfo->nparts; j++) + { + int subpartidx = pinfo->subpartindex[j]; + + Assert(subpartidx < list_length(partitionpruneinfo)); + + if (subpartidx >= 0) + partrelprune->subpartprune[j] = &partrelprunes[subpartidx]; + else + partrelprune->subpartprune[j] = NULL; + } + + rel = relation_open(pinfo->reloid, NoLock); + + partkey = RelationGetPartitionKey(rel); + partdesc = RelationGetPartitionDesc(rel); + + context->strategy = partkey->strategy; + context->partnatts = partnatts = partkey->partnatts; + + context->partopcintype = partkey->partopcintype; + context->partopfamily = partkey->partopfamily; + context->partcollation = partkey->partcollation; + context->partsupfunc = partkey->partsupfunc; + context->nparts = pinfo->nparts; + context->boundinfo = partition_bounds_copy(partdesc->boundinfo, partkey); + + context->planstate = planstate; + context->safeparams = NULL; /* empty for now */ + + partrelprune->prunesteps = pinfo->prunesteps; + + partrelprune->extparams = bms_copy(pinfo->extparams); + partrelprune->allparams = bms_union(pinfo->extparams, + pinfo->execparams); + + partprune->extparams = bms_add_members(partprune->extparams, + pinfo->extparams); + + partprune->execparams = bms_add_members(partprune->execparams, + pinfo->execparams); + + relation_close(rel, NoLock); + + i++; + } + + /* + * Cache the union of the Param ids of both types. This saves having to + * recalculate it everytime we need to know what they are. + */ + partprune->allparams = bms_union(partprune->extparams, + partprune->execparams); + + return partprune; +} + +/* + * ExecFindInitialMatchingSubPlans + * Determine which subset of subplan nodes we need to initialize based + * on the details stored in 'partprune'. Here we only determine the + * matching partitions using values known during plan startup, which is + * only external Params. Exec Params will be unknown at this time. We + * must delay pruning using exec Params until the actual executor run. + * + * It is expected that callers of this function do so once during their init + * plan. The caller must only initialize the subnodes which are returned by + * this function. The remaining subnodes should be discarded. Once this + * function has been called, future calls to ExecFindMatchingSubPlans will + * return its matching subnode indexes assuming that the caller discarded + * the original non-matching subnodes. + * + * This function must only be called if 'partprune' has any extparams. + * + * 'nsubplans' must be passed as the total number of unpruned subplans. + */ +Bitmapset * +ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, int nsubplans) +{ + PartitionedRelPruning *partrelprune; + MemoryContext oldcontext; + Bitmapset *result = NULL; + + /* + * Ensure there's actually external params, or we've not been called + * already. + */ + Assert(!bms_is_empty(partprune->extparams)); + + partrelprune = partprune->partrelpruning; + + /* + * Switch to a temp context to avoid leaking memory in the + * executor's memory context. + */ + oldcontext = MemoryContextSwitchTo(partprune->prune_context); + + /* Determine which subplans match these external params */ + find_subplans_for_extparams_recurse(partrelprune, &result); + + MemoryContextSwitchTo(oldcontext); + + /* Move to the correct memory context */ + result = bms_copy(result); + + MemoryContextReset(partprune->prune_context); + + /* + * Record that partition pruning has been performed for external params. + * This partly also serves to ensure we never call this function twice + * with the same input and also so that ExecFindMatchingSubPlans is aware + * that pruning has already been done for external Params. + */ + bms_free(partprune->extparams); + partprune->extparams = NULL; + + /* + * If any subplans were pruned, we must re-sequence the subplan indexes so + * that ExecFindMatchingSubPlans properly returns the indexes from the + * subplans which will remain after execution of this function. + */ + if (bms_num_members(result) < nsubplans) + { + int *subplanidxmap; + int i; + int newidx; + + /* + * First we must build a map which allows us to map the old subplan + * index into the new one. + */ + subplanidxmap = (int *) palloc(sizeof(int) * nsubplans); + newidx = 0; + for (i = 0; i < nsubplans; i++) + { + if (bms_is_member(i, result)) + subplanidxmap[i] = newidx++; + else + subplanidxmap[i] = -1; /* Newly pruned */ + } + + /* + * Now we can re-sequence each PartitionPruneInfo's subnodeindex + * so that they point to the new index of the subnode. + */ + for (i = 0; i < partprune->npartrelpruning; i++) + { + PartitionedRelPruning *partrelprune; + int j; + + partrelprune = &partprune->partrelpruning[i]; + + /* + * We also need to reset the allpartindexes field so that it + * only contains partition indexes that we actually still have + * subnodeindexes for. It seems easier to build a fresh one, + * rather than trying to update the existing one. + */ + bms_free(partrelprune->allpartindexes); + partrelprune->allpartindexes = NULL; + + for (j = 0; j < partrelprune->nparts; j++) + { + int oldidx = partrelprune->subnodeindex[j]; + + /* + * If this partition existed as a subplan then change the old + * subplan index to the new subplan index. The new index may + * become -1 if the partition was pruned above, or it may just + * come earlier in the subplan list due to some subplans + * removed earlier in the list. + */ + if (oldidx >= 0) + { + partrelprune->subnodeindex[j] = subplanidxmap[oldidx]; + + if (subplanidxmap[oldidx] >= 0) + partrelprune->allpartindexes = + bms_add_member(partrelprune->allpartindexes, + j); + } + } + } + + pfree(subplanidxmap); + } + + + return result; +} + +/* + * find_subplans_for_extparams_recurse + * Recursive worker function for ExecFindInitialMatchingSubPlans. + */ +static void +find_subplans_for_extparams_recurse(PartitionedRelPruning *partrelprune, + Bitmapset **validsubplans) +{ + PartitionPruneContext *context = &partrelprune->context; + Bitmapset *partset; + int i; + + /* Guard against stack overflow due to overly deep partition hierarchy. */ + check_stack_depth(); + + /* + * We only need to determine the matching partitions if there are any + * ext params matching the partition key at this level. If there are no + * such params, then we can simply return all subnodes which belong to + * this parent partition. The planner should have already determined + * these to be the minimum possible set. We must still recursively visit + * any subpartitioned tables as we may find their partition keys match + * some Params at their level. + */ + if (!bms_is_empty(partrelprune->extparams)) + { + context->safeparams = partrelprune->extparams; + partset = get_matching_partitions(context, partrelprune->prunesteps); + } + else + partset = partrelprune->allpartindexes; + + /* Translate partset into subnode indexes */ + i = -1; + while ((i = bms_next_member(partset, i)) >= 0) + { + if (partrelprune->subnodeindex[i] >= 0) + *validsubplans = bms_add_member(*validsubplans, + partrelprune->subnodeindex[i]); + else if (partrelprune->subpartprune[i] != NULL) + find_subplans_for_extparams_recurse(partrelprune->subpartprune[i], + validsubplans); + else + { + /* + * If this happens then we're somehow missing a subnode. This + * shouldn't happen and could only happen if a more restrictive + * clause list was used for partition elimination during planning + * than what was used here. + */ + elog(ERROR, "partition missing from subplans"); + } + } +} + +/* + * ExecFindMatchingSubPlans + * Determine which subplans match the the pruning steps detailed in + * 'partprune' for the current Param values. + */ +Bitmapset * +ExecFindMatchingSubPlans(PartitionPruning *partprune) +{ + PartitionedRelPruning *partrelprune; + MemoryContext oldcontext; + Bitmapset *result = NULL; + + partrelprune = partprune->partrelpruning; + + /* + * Switch to a temp context to avoid leaking memory in the + * executor's memory context. + */ + oldcontext = MemoryContextSwitchTo(partprune->prune_context); + + find_subplans_for_allparams_recurse(partrelprune, &result); + + MemoryContextSwitchTo(oldcontext); + + /* Move to the correct memory context */ + result = bms_copy(result); + + MemoryContextReset(partprune->prune_context); + + return result; +} + +/* + * find_subplans_for_allparams_recurse + * Recursive worker function for ExecFindMatchingSubPlans. + */ +static void +find_subplans_for_allparams_recurse(PartitionedRelPruning *partrelprune, + Bitmapset **validsubplans) +{ + PartitionPruneContext *context = &partrelprune->context; + Bitmapset *partset; + int i; + + /* Guard against stack overflow due to overly deep partition hierarchy. */ + check_stack_depth(); + + /* + * We only need to determine the matching partitions if there are any + * params matching the partition key at this level. If there are no + * matching params, then we can simply return all subnodes which belong + * to this parent partition. The planner should have already determined + * these to be the minimum possible set. We must still recursively visit + * any subpartitioned tables as we may find their partition keys match + * some Params at their level. + */ + if (!bms_is_empty(partrelprune->allparams)) + { + context->safeparams = partrelprune->allparams; + partset = get_matching_partitions(context, partrelprune->prunesteps); + } + else + partset = partrelprune->allpartindexes; + + /* Translate partset into subnode indexes */ + i = -1; + while ((i = bms_next_member(partset, i)) >= 0) + { + if (partrelprune->subnodeindex[i] >= 0) + *validsubplans = bms_add_member(*validsubplans, + partrelprune->subnodeindex[i]); + else if (partrelprune->subpartprune[i] != NULL) + find_subplans_for_allparams_recurse(partrelprune->subpartprune[i], + validsubplans); + else + { + /* + * If this happens then we're somehow missing a subnode. This + * shouldn't happen and could only happen if a more restrictive + * clause list was used for partition elimination during planning + * than what was used here. + */ + elog(ERROR, "partition missing from subplans"); + } + } +} diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 22ed053532..d63fcf782e 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2164,6 +2164,23 @@ _copyPartitionPruneStepCombine(const PartitionPruneStepCombine *from) return newnode; } +static PartitionPruneInfo * +_copyPartitionPruneInfo(const PartitionPruneInfo *from) +{ + PartitionPruneInfo *newnode = makeNode(PartitionPruneInfo); + + COPY_SCALAR_FIELD(reloid); + COPY_NODE_FIELD(prunesteps); + COPY_BITMAPSET_FIELD(allpartindexes); + COPY_SCALAR_FIELD(nparts); + COPY_POINTER_FIELD(subnodeindex, from->nparts * sizeof(int)); + COPY_POINTER_FIELD(subpartindex, from->nparts * sizeof(int)); + COPY_BITMAPSET_FIELD(extparams); + COPY_BITMAPSET_FIELD(execparams); + + return newnode; +} + /* **************************************************************** * relation.h copy functions * @@ -5068,6 +5085,9 @@ copyObjectImpl(const void *from) case T_PlaceHolderInfo: retval = _copyPlaceHolderInfo(from); break; + case T_PartitionPruneInfo: + retval = _copyPartitionPruneInfo(from); + break; /* * VALUE NODES diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 3f9e2585c7..bd62f81cea 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1730,6 +1730,30 @@ _outOnConflictExpr(StringInfo str, const OnConflictExpr *node) WRITE_NODE_FIELD(exclRelTlist); } +static void +_outPartitionPruneInfo(StringInfo str, const PartitionPruneInfo *node) +{ + int i; + + WRITE_NODE_TYPE("PARTITIONPRUNEINFO"); + + WRITE_OID_FIELD(reloid); + WRITE_NODE_FIELD(prunesteps); + WRITE_BITMAPSET_FIELD(allpartindexes); + WRITE_INT_FIELD(nparts); + + appendStringInfoString(str, " :subnodeindex"); + for (i = 0; i < node->nparts; i++) + appendStringInfo(str, " %d", node->subnodeindex[i]); + + appendStringInfoString(str, " :subpartindex"); + for (i = 0; i < node->nparts; i++) + appendStringInfo(str, " %d", node->subpartindex[i]); + + WRITE_BITMAPSET_FIELD(extparams); + WRITE_BITMAPSET_FIELD(execparams); +} + /***************************************************************************** * * Stuff from relation.h. @@ -3952,6 +3976,9 @@ outNode(StringInfo str, const void *obj) case T_PartitionPruneStepCombine: _outPartitionPruneStepCombine(str, obj); break; + case T_PartitionPruneInfo: + _outPartitionPruneInfo(str, obj); + break; case T_Path: _outPath(str, obj); break; diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 8348933151..5200f3cd8a 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -1354,6 +1354,23 @@ _readPartitionPruneStepCombine(void) READ_DONE(); } +static PartitionPruneInfo * +_readPartitionPruneInfo(void) +{ + READ_LOCALS(PartitionPruneInfo); + + READ_OID_FIELD(reloid); + READ_NODE_FIELD(prunesteps); + READ_BITMAPSET_FIELD(allpartindexes); + READ_INT_FIELD(nparts); + READ_INT_ARRAY(subnodeindex, local_node->nparts); + READ_INT_ARRAY(subpartindex, local_node->nparts); + READ_BITMAPSET_FIELD(extparams); + READ_BITMAPSET_FIELD(execparams); + + READ_DONE(); +} + /* * Stuff from parsenodes.h. */ @@ -2602,6 +2619,8 @@ parseNodeString(void) return_value = _readPartitionPruneStepOp(); else if (MATCH("PARTITIONPRUNESTEPCOMBINE", 25)) return_value = _readPartitionPruneStepCombine(); + else if (MATCH("PARTITIONPRUNEINFO", 18)) + return_value = _readPartitionPruneInfo(); else if (MATCH("RTE", 3)) return_value = _readRangeTblEntry(); else if (MATCH("RANGETBLFUNCTION", 16)) diff --git a/src/backend/optimizer/util/partprune.c b/src/backend/optimizer/util/partprune.c index e5e6d7530b..45463d7ed8 100644 --- a/src/backend/optimizer/util/partprune.c +++ b/src/backend/optimizer/util/partprune.c @@ -32,6 +32,7 @@ #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/partprune.h" +#include "optimizer/pathnode.h" #include "optimizer/planner.h" #include "optimizer/predtest.h" #include "optimizer/prep.h" @@ -85,6 +86,7 @@ typedef struct GeneratePruningStepsContext List *steps; } GeneratePruningStepsContext; +static bool pull_partkey_params(PartitionPruneInfo *pinfo, List *steps); static List *generate_partition_pruning_steps_internal(RelOptInfo *rel, GeneratePruningStepsContext *context, List *clauses, @@ -169,6 +171,10 @@ prune_append_rel_partitions(RelOptInfo *rel) context.nparts = rel->nparts; context.boundinfo = rel->boundinfo; + /* Not valid when being called from the planner */ + context.planstate = NULL; + context.safeparams = NULL; + partindexes = get_matching_partitions(&context, pruning_steps); /* Add selected partitions' RT indexes to result. */ @@ -232,8 +238,259 @@ generate_partition_pruning_steps(RelOptInfo *rel, List *clauses, return context.steps; } +/* + * make_partition_pruneinfo + * Return a List of PartitionPruneInfos, one for each 'partitioned_rel', + * or NIL if no Params were found matching the partition key, in which + * case run-time partition pruning is useless. + * + * Here we index the subpaths by partition index so that we're able to + * translate the output of get_matching_partitions into subpath indexes to + * possibly allow for further partition pruning to be performed during + * execution. + */ +List * +make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, + List *resultRelations, List *subpaths, + List *prunequal) +{ + RangeTblEntry *rte; + RelOptInfo *parentpart; + ListCell *lc; + List *pinfolist = NIL; + int *allsubnodeindex; + int *allsubpartindex; + int i; + bool gotparam = false; + + /* + * Allocate two arrays, one to allow quick lookups of the 'subpaths' index + * of a relation by relid and another to lookup the 'partitioned_rel' + * index by relid. + */ + allsubnodeindex = palloc(sizeof(int) * root->simple_rel_array_size); + allsubpartindex = palloc(sizeof(int) * root->simple_rel_array_size); + + /* Initialize to -1 to indicate the rel was not found */ + for (i = 0; i < root->simple_rel_array_size; i++) + { + allsubnodeindex[i] = -1; + allsubpartindex[i] = -1; + } + + /* + * Now loop over each subpath and fill in the index of the subpath for the + * subpath's relid. + */ + if (resultRelations != NIL) + { + i = 0; + foreach(lc, resultRelations) + { + int resultrel = lfirst_int(lc); + Assert(resultrel < root->simple_rel_array_size); + allsubnodeindex[resultrel] = i; + i++; + } + } + else + { + i = 0; + foreach(lc, subpaths) + { + Path *path = (Path *) lfirst(lc); + RelOptInfo *pathrel = path->parent; + + Assert(IS_SIMPLE_REL(pathrel)); + Assert(pathrel->relid < root->simple_rel_array_size); + + allsubnodeindex[pathrel->relid] = i; + i++; + } + } + + /* Likewise for the partition_rels */ + i = 0; + foreach(lc, partition_rels) + { + Index rti = lfirst_int(lc); + + Assert(rti < root->simple_rel_array_size); + + allsubpartindex[rti] = i; + i++; + } + + /* We now build a PartitionPruneInfo for each partition_rels */ + i = 0; + foreach(lc, partition_rels) + { + Index rti = lfirst_int(lc); + RelOptInfo *subpart = find_base_rel(root, rti); + PartitionPruneInfo *pinfo; + int nparts = subpart->nparts; + int *subnodeindex; + int *subpartindex; + List *partprunequal; + bool constfalse; + + rte = root->simple_rte_array[subpart->relid]; + + pinfo = makeNode(PartitionPruneInfo); + pinfo->reloid = rte->relid; + + /* + * The first item in the list is the target partitioned relation. The + * quals belong to this relation, so require no translation. + */ + if (i == 0) + { + parentpart = subpart; + partprunequal = prunequal; + } + else + { + /* + * For sub-partitioned tables the columns may not be in the same + * order as the parent, so we must translate the prunequal to make + * it compatible with this relation. + */ + partprunequal = (List *) + adjust_appendrel_attrs_multilevel(root, + (Node *) prunequal, + subpart->relids, + parentpart->relids); + } + + pinfo->prunesteps = generate_partition_pruning_steps(subpart, + partprunequal, + &constfalse); + + if (constfalse) + { + /* + * This shouldn't happen as the planner should have detected this + * earlier. However, we do use additional quals from parameterized + * paths here. These do only compare Params to the partition key, + * so this shouldn't cause the discovery of any new qual + * contradictions that were not previously discovered. We'd + * better do something sane here anyway, so let's just disable + * run-time pruning. + */ + return NIL; + } + + pinfo->allpartindexes = NULL; + pinfo->nparts = nparts; + pinfo->subnodeindex = subnodeindex = palloc(nparts * sizeof(int)); + pinfo->subpartindex = subpartindex = palloc(nparts * sizeof(int)); + pinfo->extparams = NULL; + pinfo->execparams = NULL; + + /* + * Extract Params matching partition key and record if we got any. + * We'll not bother enabling run-time pruning if no params matched + * the partition key at any level of partitioning. + */ + gotparam |= pull_partkey_params(pinfo, pinfo->prunesteps); + + /* + * Loop over each partition of the partitioned rel and record the + * subpath index for each. Any partitions which are not present + * in the subpaths List will be set to -1, and any subpartitioned + * table which is not present will also be set to -1. + */ + for (i = 0; i < nparts; i++) + { + RelOptInfo *partrel = subpart->part_rels[i]; + int subnodeidx = allsubnodeindex[partrel->relid]; + int subpartidx = allsubpartindex[partrel->relid]; + + subnodeindex[i] = subnodeidx; + subpartindex[i] = subpartidx; + + /* + * Record the indexes of all the partition indexes that we have + * subnodes or subparts for. This allows an optimization to skip + * attempting any run-time pruning when no Params are found + * matching the partition key at this level. + */ + if (subnodeidx >= 0 || subpartidx >= 0) + pinfo->allpartindexes = bms_add_member(pinfo->allpartindexes, + i); + } + + pinfolist = lappend(pinfolist, pinfo); + i++; + } + + pfree(allsubnodeindex); + pfree(allsubpartindex); + + if (gotparam) + return pinfolist; + + /* + * If no Params were found to match the partition key on any of the + * partitioned relations then there's no point doing any run-time + * partition pruning. + */ + return NIL; +} + /* Module-local functions */ +/* + * pull_partkey_params + * Loop through each pruning step and record each external and exec + * Params being compared to the partition keys. + */ +static bool +pull_partkey_params(PartitionPruneInfo *pinfo, List *steps) +{ + ListCell *lc; + bool gotone = false; + + foreach(lc, steps) + { + PartitionPruneStepOp *stepop = lfirst(lc); + ListCell *lc2; + + if (!IsA(stepop, PartitionPruneStepOp)) + continue; + + foreach(lc2, stepop->exprs) + { + Expr *expr = lfirst(lc2); + + if (IsA(expr, Param)) + { + Param *param = (Param *) expr; + + switch (param->paramkind) + { + case PARAM_EXTERN: + pinfo->extparams = bms_add_member(pinfo->extparams, + param->paramid); + break; + case PARAM_EXEC: + pinfo->execparams = bms_add_member(pinfo->execparams, + param->paramid); + break; + + default: + elog(ERROR, "unrecognized paramkind: %d", + (int) param->paramkind); + break; + } + gotone = true; + } + } + } + + return gotone; +} + /* * generate_partition_pruning_steps_internal * Processes 'clauses' to generate partition pruning steps. diff --git a/src/include/catalog/partition.h b/src/include/catalog/partition.h index 8981901272..558a996ac3 100644 --- a/src/include/catalog/partition.h +++ b/src/include/catalog/partition.h @@ -16,6 +16,7 @@ #include "fmgr.h" #include "executor/tuptable.h" #include "nodes/execnodes.h" +#include "nodes/relation.h" #include "parser/parse_node.h" #include "utils/rel.h" @@ -62,6 +63,18 @@ typedef struct PartitionPruneContext /* Partition boundary info */ PartitionBoundInfo boundinfo; + + /* + * Can be set when the context is used from the executor to allow + * resolution of Param values. + */ + PlanState *planstate; + + /* + * Parameters that are safe to be used for partition pruning. execparams + * are not safe to use until after init plan. + */ + Bitmapset *safeparams; } PartitionPruneContext; extern void RelationBuildPartitionDesc(Relation relation); diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index 03a599ad57..63e287ea81 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -17,6 +17,7 @@ #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "nodes/plannodes.h" +#include "optimizer/partprune.h" /*----------------------- * PartitionDispatch - information about one partitioned table in a partition @@ -108,6 +109,77 @@ typedef struct PartitionTupleRouting TupleTableSlot *root_tuple_slot; } PartitionTupleRouting; +/*----------------------- + * PartitionedRelPruning - Encapsulates all information required to support + * elimination of partitions in node types which support arbitrary Lists of + * subplans. Information stored here allows partprune.c's partition pruning + * functions to be called and the return value of partition indexes translated + * into the subpath indexes of node types such as Append, thus allowing us to + * bypass certain subnodes when we have proofs that indicate that no tuple + * matching the 'prunesteps' will be found within. + * + * nparts The number of partitions which belong to this + * partitioned relation. Also defines the size of + * the 'subnodeindex' and 'subpartprune' arrays. + * subnodeindex An array of nparts containing the subnode + * index which matches this partition index, or + * -1 if there is no match. + * subpartprune An array of nparts containing the + * PartitionedRelPruning details this partition + * index for sub-partitioned tables. + * allpartindexes A Bitmapset of the partition index that we have + * subnodes mapped for. + * belong to this partition. + * context Contains the context details required to call + * the partition pruning code. + * prunesteps Contains list of PartitionPruneStep used to + * perform the actual pruning. + *----------------------- + */ +typedef struct PartitionedRelPruning +{ + int nparts; + int *subnodeindex; + struct PartitionedRelPruning **subpartprune; + Bitmapset *allpartindexes; + PartitionPruneContext context; + List *prunesteps; + Bitmapset *extparams; + Bitmapset *allparams; +} PartitionedRelPruning; + +/*----------------------- + * PartitionPruning - Encapsulates a hierarchy of PartitionedRelPruning + * structs and also stores all Param IDs which were found to match the + * partition keys of each partition. This struct can be attached to node + * types which support arbitrary Lists of subnodes containing partitions to + * allow subnodes to be eliminated due to the clauses being unable to match + * to any tuple that the subnode could possibly produce. + * + * partrelpruning Array of PartitionedRelPruning for the node's target + * partitioned relation. First element contains the + * details for the target partitioned table. + * npartrelpruning Number of items in partrelpruning array. + * prune_context A memory context which can be used to call the query + * planner's partition prune functions. + * extparams All PARAM_EXTERN Param IDs which were found to match a + * partition key in each of the contained + * PartitionedRelPruning structs. + * execparams As above but for PARAM_EXEC. + * allparams Union of extparams and execparams, saved to avoid + * recalculation. + *----------------------- + */ +typedef struct PartitionPruning +{ + PartitionedRelPruning *partrelpruning; + int npartrelpruning; + MemoryContext prune_context; + Bitmapset *extparams; + Bitmapset *execparams; + Bitmapset *allparams; +} PartitionPruning; + extern PartitionTupleRouting *ExecSetupPartitionTupleRouting(ModifyTableState *mtstate, Relation rel); extern int ExecFindPartition(ResultRelInfo *resultRelInfo, @@ -126,5 +198,10 @@ extern HeapTuple ConvertPartitionTupleSlot(TupleConversionMap *map, TupleTableSlot *new_slot, TupleTableSlot **p_my_slot); extern void ExecCleanupTupleRouting(PartitionTupleRouting *proute); +extern PartitionPruning *ExecSetupPartitionPruning(PlanState *planstate, + List *partitionpruneinfo); +extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruning *partprune); +extern Bitmapset *ExecFindInitialMatchingSubPlans(PartitionPruning *partprune, + int nsubplans); #endif /* EXECPARTITION_H */ diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index e6b5770c74..adb159a6da 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -195,6 +195,7 @@ typedef enum NodeTag T_PartitionPruneStep, T_PartitionPruneStepOp, T_PartitionPruneStepCombine, + T_PartitionPruneInfo, /* * TAGS FOR EXPRESSION STATE NODES (execnodes.h) diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h index a71d729e72..c9dad76755 100644 --- a/src/include/nodes/primnodes.h +++ b/src/include/nodes/primnodes.h @@ -1580,4 +1580,27 @@ typedef struct PartitionPruneStepCombine List *source_stepids; } PartitionPruneStepCombine; +/*---------- + * PartitionPruneInfo - Details required to allow the executor to prune + * partitions. + * + * Here we store mapping details to allow translation of a partitioned table's + * index into subnode indexes for node types which support arbitrary numbers + * of sub nodes, such as Append. + *---------- + */ +typedef struct PartitionPruneInfo +{ + NodeTag type; + Oid reloid; /* Oid of partition rel */ + List *prunesteps; /* List of PartitionPruneStep */ + Bitmapset *allpartindexes; /* All part index we have subnodes for at this + * level */ + int nparts; /* length of the following arrays */ + int *subnodeindex; /* subnode index indexed by partition id */ + int *subpartindex; /* subpart index indexed by partition id */ + Bitmapset *extparams; /* All external ParamIDs seen in prunesteps */ + Bitmapset *execparams; /* All exec ParamIDs seen in prunesteps */ +} PartitionPruneInfo; + #endif /* PRIMNODES_H */ diff --git a/src/include/optimizer/partprune.h b/src/include/optimizer/partprune.h index 1f2fe297a3..b7352d150c 100644 --- a/src/include/optimizer/partprune.h +++ b/src/include/optimizer/partprune.h @@ -20,4 +20,8 @@ extern Relids prune_append_rel_partitions(RelOptInfo *rel); extern List *generate_partition_pruning_steps(RelOptInfo *rel, List *clauses, bool *constfalse); +extern List *make_partition_pruneinfo(PlannerInfo *root, List *partition_rels, + List *resultRelations, List *subpaths, + List *prunequal); + #endif /* PARTPRUNE_H */ -- 2.16.2.windows.1