From 509121ddd1d72b49773fde19dc89ae1a1295a6f2 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Thu, 17 Jan 2019 09:11:10 -0500 Subject: [PATCH v3 3/4] Teach runtime partition pruning to cope with concurrent partition adds. If new partitions were added between plan time and execution time, the indexes stored in the subplan_map[] and subpart_map[] arrays within the plan's PartitionedRelPruneInfo would no longer be correct. Adjust the code to cope with added partitions. There does not seem to be a simple way to cope with partitions that are removed, mostly because they could then get added back again with different bounds, so don't try to cope with that situation. --- src/backend/executor/execPartition.c | 68 +++++++++++++++++++++++----- src/backend/nodes/copyfuncs.c | 1 + src/backend/nodes/outfuncs.c | 1 + src/backend/nodes/readfuncs.c | 1 + src/backend/partitioning/partprune.c | 7 ++- src/include/nodes/plannodes.h | 1 + 6 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index db133b37a5..de84d03680 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -1599,18 +1599,6 @@ ExecCreatePartitionPruneState(PlanState *planstate, int n_steps; ListCell *lc3; - /* - * We must copy the subplan_map rather than pointing directly to - * the plan's version, as we may end up making modifications to it - * later. - */ - pprune->subplan_map = palloc(sizeof(int) * pinfo->nparts); - memcpy(pprune->subplan_map, pinfo->subplan_map, - sizeof(int) * pinfo->nparts); - - /* We can use the subpart_map verbatim, since we never modify it */ - pprune->subpart_map = pinfo->subpart_map; - /* present_parts is also subject to later modification */ pprune->present_parts = bms_copy(pinfo->present_parts); @@ -1625,6 +1613,62 @@ ExecCreatePartitionPruneState(PlanState *planstate, partdesc = PartitionDirectoryLookup(estate->es_partition_directory, partrel); + /* + * Initialize the subplan_map and subpart_map. Since detaching a + * partition requires AccessExclusiveLock, no partitions can have + * disappeared, nor can the bounds for any partition have changed. + * However, new partitions may have been added. + */ + Assert(partdesc->nparts >= pinfo->nparts); + pprune->subplan_map = palloc(sizeof(int) * partdesc->nparts); + if (partdesc->nparts == pinfo->nparts) + { + /* + * There are no new partitions, so this is simple. We can + * simply point to the subpart_map from the plan, but we must + * copy the subplan_map since we may change it later. + */ + pprune->subpart_map = pinfo->subpart_map; + memcpy(pprune->subplan_map, pinfo->subplan_map, + sizeof(int) * pinfo->nparts); + + /* Double-check that list of relations has not changed. */ + Assert(memcmp(partdesc->oids, pinfo->relid_map, + pinfo->nparts * sizeof(Oid)) == 0); + } + else + { + int pd_idx = 0; + int pp_idx; + + /* + * Some new partitions have appeared since plan time, and + * those are reflected in our PartitionDesc but were not + * present in the one used to construct subplan_map and + * subpart_map. So we must construct new and longer arrays + * where the partitions that were originally present map to the + * same place, and any added indexes map to -1, as if the + * new partitions had been pruned. + */ + pprune->subpart_map = palloc(sizeof(int) * partdesc->nparts); + for (pp_idx = 0; pp_idx < partdesc->nparts; ++pp_idx) + { + if (pinfo->relid_map[pd_idx] != partdesc->oids[pp_idx]) + { + pprune->subplan_map[pp_idx] = -1; + pprune->subpart_map[pp_idx] = -1; + } + else + { + pprune->subplan_map[pp_idx] = + pinfo->subplan_map[pd_idx]; + pprune->subpart_map[pp_idx] = + pinfo->subpart_map[pd_idx++]; + } + } + Assert(pd_idx == pinfo->nparts); + } + n_steps = list_length(pinfo->pruning_steps); context->strategy = partkey->strategy; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index e15724bb0e..d5fddce953 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -1197,6 +1197,7 @@ _copyPartitionedRelPruneInfo(const PartitionedRelPruneInfo *from) COPY_SCALAR_FIELD(nexprs); COPY_POINTER_FIELD(subplan_map, from->nparts * sizeof(int)); COPY_POINTER_FIELD(subpart_map, from->nparts * sizeof(int)); + COPY_POINTER_FIELD(relid_map, from->nparts * sizeof(int)); COPY_POINTER_FIELD(hasexecparam, from->nexprs * sizeof(bool)); COPY_SCALAR_FIELD(do_initial_prune); COPY_SCALAR_FIELD(do_exec_prune); diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 65302fe65b..65b4a63013 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -947,6 +947,7 @@ _outPartitionedRelPruneInfo(StringInfo str, const PartitionedRelPruneInfo *node) WRITE_INT_FIELD(nexprs); WRITE_INT_ARRAY(subplan_map, node->nparts); WRITE_INT_ARRAY(subpart_map, node->nparts); + WRITE_OID_ARRAY(relid_map, node->nparts); WRITE_BOOL_ARRAY(hasexecparam, node->nexprs); WRITE_BOOL_FIELD(do_initial_prune); WRITE_BOOL_FIELD(do_exec_prune); diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 5aa42242a9..fc60b0a7c5 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -2385,6 +2385,7 @@ _readPartitionedRelPruneInfo(void) READ_INT_FIELD(nexprs); READ_INT_ARRAY(subplan_map, local_node->nparts); READ_INT_ARRAY(subpart_map, local_node->nparts); + READ_OID_ARRAY(relid_map, local_node->nparts); READ_BOOL_ARRAY(hasexecparam, local_node->nexprs); READ_BOOL_FIELD(do_initial_prune); READ_BOOL_FIELD(do_exec_prune); diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index 8c9721935d..b5c0889935 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -47,8 +47,9 @@ #include "optimizer/appendinfo.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" -#include "partitioning/partprune.h" +#include "parser/parsetree.h" #include "partitioning/partbounds.h" +#include "partitioning/partprune.h" #include "rewrite/rewriteManip.h" #include "utils/lsyscache.h" @@ -359,6 +360,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, int partnatts = subpart->part_scheme->partnatts; int *subplan_map; int *subpart_map; + Oid *relid_map; List *partprunequal; List *pruning_steps; bool contradictory; @@ -434,6 +436,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, */ subplan_map = (int *) palloc(nparts * sizeof(int)); subpart_map = (int *) palloc(nparts * sizeof(int)); + relid_map = (Oid *) palloc(nparts * sizeof(int)); present_parts = NULL; for (i = 0; i < nparts; i++) @@ -444,6 +447,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, subplan_map[i] = subplanidx; subpart_map[i] = subpartidx; + relid_map[i] = planner_rt_fetch(partrel->relid, root)->relid; if (subplanidx >= 0) { present_parts = bms_add_member(present_parts, i); @@ -462,6 +466,7 @@ make_partitionedrel_pruneinfo(PlannerInfo *root, RelOptInfo *parentrel, pinfo->nparts = nparts; pinfo->subplan_map = subplan_map; pinfo->subpart_map = subpart_map; + pinfo->relid_map = relid_map; /* Determine which pruning types should be enabled at this level */ doruntimeprune |= analyze_partkey_exprs(pinfo, pruning_steps, diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 6d087c268f..d66a187a53 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -1108,6 +1108,7 @@ typedef struct PartitionedRelPruneInfo int nexprs; /* Length of hasexecparam[] */ int *subplan_map; /* subplan index by partition index, or -1 */ int *subpart_map; /* subpart index by partition index, or -1 */ + Oid *relid_map; /* relation OID by partition index, or -1 */ bool *hasexecparam; /* true if corresponding pruning_step contains * any PARAM_EXEC Params. */ bool do_initial_prune; /* true if pruning should be performed -- 2.17.2 (Apple Git-113)